{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>116</td>\n",
       "      <td>74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25.6</td>\n",
       "      <td>0.201</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3</td>\n",
       "      <td>78</td>\n",
       "      <td>50</td>\n",
       "      <td>32</td>\n",
       "      <td>88</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.248</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10</td>\n",
       "      <td>115</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35.3</td>\n",
       "      <td>0.134</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>197</td>\n",
       "      <td>70</td>\n",
       "      <td>45</td>\n",
       "      <td>543</td>\n",
       "      <td>30.5</td>\n",
       "      <td>0.158</td>\n",
       "      <td>53</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8</td>\n",
       "      <td>125</td>\n",
       "      <td>96</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.232</td>\n",
       "      <td>54</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "5            5      116             74              0        0  25.6   \n",
       "6            3       78             50             32       88  31.0   \n",
       "7           10      115              0              0        0  35.3   \n",
       "8            2      197             70             45      543  30.5   \n",
       "9            8      125             96              0        0   0.0   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  \n",
       "5                     0.201   30        0  \n",
       "6                     0.248   26        1  \n",
       "7                     0.134   29        0  \n",
       "8                     0.158   53        1  \n",
       "9                     0.232   54        1  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "from matplotlib import pyplot\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "from sklearn.linear_model import RidgeCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import  GridSearchCV\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "data_path = \"C:/Users/Administrator/Desktop/AI/Work/HW2/\"\n",
    "\n",
    "data = pd.read_csv(data_path +\"diabetes.csv\")\n",
    "\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 768 non-null int64\n",
      "Glucose                     768 non-null int64\n",
      "BloodPressure               768 non-null int64\n",
      "SkinThickness               768 non-null int64\n",
      "Insulin                     768 non-null int64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null int64\n",
      "Outcome                     768 non-null int64\n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAE45JREFUeJzt3X20ZXV93/H3BwY0RcLwMLIIw2Sw\nmWSFVRVxpKiND2hb0ZZBKnmoK0zNrE5c9QEXaQqmqZjYGk00WpuGdFagQptIkIqMSqJkBGxMIQyK\nyIMuRorMZCwz8oxULPLtH+d34TjsuXfPwL7ncO/7tdZZZ+/f/u1zvnet4XzY+7f3b6eqkCRpV/tM\nugBJ0nQyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdVoy6QKeisMOO6xWrlw5\n6TIk6Rnl+uuv/25VLZur3zM6IFauXMnmzZsnXYYkPaMk+Xaffp5ikiR1MiAkSZ0MCElSJwNCktTJ\ngJAkdRo0IJLckeTrSW5Isrm1HZLkiiS3tfeDW3uSfCzJliQ3JjluyNokSbObjyOIV1fVsVW1uq2f\nDWyqqlXAprYOcBKwqr3WA+fOQ22SpN2YxCmmNcAFbfkC4JSx9gtr5BpgaZIjJlCfJInhA6KALyS5\nPsn61nZ4VX0HoL0/t7UfCWwd23dba5MkTcDQd1K/vKq2J3kucEWSb8zSNx1t9aROo6BZD7BixYqn\nXOCLf/3Cp/wZWniu/73TJ12CNHGDHkFU1fb2vgO4FDgeuGvm1FF739G6bwOOGtt9ObC94zM3VNXq\nqlq9bNmcU4lIkvbSYAGR5IAkB84sA/8IuAnYCKxt3dYCl7XljcDp7WqmE4D7Z05FSZLm35CnmA4H\nLk0y8z1/WlV/keQ64OIk64A7gdNa/8uB1wNbgIeBtwxYmyRpDoMFRFXdDrywo/1u4DUd7QW8bah6\nJEl7xjupJUmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJ\ngJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJ\ngJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1Gjwgkuyb5KtJPtvWj05y\nbZLbkvxZkv1b+7Pa+pa2feXQtUmSdm8+jiDOAG4dW/8g8JGqWgXcC6xr7euAe6vqp4CPtH6SpAkZ\nNCCSLAfeAPxxWw9wInBJ63IBcEpbXtPWadtf0/pLkiZg6COIjwL/BnisrR8K3FdVj7b1bcCRbflI\nYCtA235/6y9JmoA5AyLJAUn2acs/neTkJPv12O+fADuq6vrx5o6u1WPb+OeuT7I5yeadO3fOVYYk\naS/1OYL4EvDsJEcCm4C3AB/vsd/LgZOT3AFcxOjU0keBpUmWtD7Lge1teRtwFEDbfhBwz64fWlUb\nqmp1Va1etmxZjzIkSXujT0Ckqh4GTgX+U1W9EThmrp2q6t1VtbyqVgK/CHyxqt4MXAm8qXVbC1zW\nlje2ddr2L1bVk44gJEnzo1dAJHkp8Gbgc61tySz953IWcGaSLYzGGM5r7ecBh7b2M4Gzn8J3SJKe\noj4/9O8C3g1cWlU3J3keo6OA3qrqKuCqtnw7cHxHn+8Dp+3J50qShjNnQFTV1cDVSQ5o67cD7xy6\nMEnSZPW5iumlSW6h3eyW5IVJ/nDwyiRJE9VnDOKjwD8G7gaoqq8BrxiyKEnS5PW6Ua6qtu7S9MMB\napEkTZE+g9Rbk7wMqDax3jv50bmVJEkLUJ8jiLcCb2M0FcY24Ni2LklawPpcxfRdRvdASJIWkT5X\nMV2QZOnY+sFJzh+2LEnSpPU5xfSCqrpvZqWq7gVeNFxJkqRp0Ccg9kly8MxKkkN4alNtSJKeAfr8\n0H8Y+OskMw/5OQ34D8OVJEmaBn0GqS9Mcj3wakbPbDi1qm4ZvDJJ0kT1PVX0DUbPj14CkGRFVd05\nWFWSpImbMyCSvAM4B7iL0R3UYfSktxcMW5okaZL6HEGcAfxMVd09dDGSpOnR5yqmrcD9QxciSZou\nfY4gbgeuSvI54JGZxqr6/cGqkiRNXJ+AuLO99m8vSdIi0Ocy198CSHJAVX1v+JIkSdPAJ8pJkjr5\nRDlJUiefKCdJ6uQT5SRJnXyinCSp06xHEEn2BX65qnyinCQtMrMeQVTVD4E181SLJGmK9BmD+HKS\nPwD+DHj8Poiq+spgVUmSJq5PQLysvf/2WFsBJz795UiSpsVcYxD7AOdW1cXzVI8kaUrMNQbxGPD2\neapFkjRF+lzmekWSf53kqCSHzLwGr0ySNFF9xiB+pb2P3/tQwPOe/nIkSdOiz2yuR89HIZKk6dLn\nmdSnd7VX1YVz7Pds4EvAs9r3XFJV5yQ5GrgIOAT4CqMb8X6Q5FnAhcCLGU0M+AtVdcce/C2SpKdR\nnzGIl4y9fg54L3Byj/0eAU6sqhcymp7jdUlOAD4IfKSqVgH3Auta/3XAvVX1U8BHWj9J0oT0OcX0\njvH1JAcB/63HfgU81Fb3a6+Z+yf+eWu/gFHgnMvoju33tvZLgD9IkvY5kqR51mu67108DKzq0zHJ\nvkluAHYAVwDfAu6rqkdbl22MJgGkvW8FaNvvBw7di/okSU+DPmMQn2H0f/4wCpRjgF43zrW5nI5N\nshS4FPjZrm4zXzXLtvF61gPrAVasWNGnDEnSXuhzmeuHxpYfBb5dVdv25Euq6r4kVwEnAEuTLGlH\nCcuB7a3bNuAoYFuSJcBBwD0dn7UB2ACwevVqTz9J0kD6nGK6E7i2qq6uqi8DdydZOddOSZa1IweS\n/BjwWkYPGroSeFPrtha4rC1vbOu07V90/EGSJqdPQHwSeGxs/YetbS5HAFcmuRG4Driiqj4LnAWc\nmWQLozGG81r/84BDW/uZwNn9/gRJ0hD6nGJaUlU/mFlp9yzsP9dOVXUj8KKO9tuB4zvavw+c1qMe\nSdI86HMEsTPJ4/c9JFkDfHe4kiRJ06DPEcRbgT9pDw2C0WBy593VkqSFo8+Nct8CTkjyHCBV9eDw\nZUmSJm3OU0xJ3p9kaVU9VFUPJjk4yb+fj+IkSZPTZwzipKq6b2alqu4FXj9cSZKkadAnIPZtM60C\nj9/T8KxZ+kuSFoA+g9T/HdiU5L8ymvriVxhNsidJWsD6DFL/brvZ7bWt6X1V9flhy5IkTVqfIwiA\nr/LEdN1fHa4cSTPu/O3nT7oETaEV7/n6vH1Xn6uYfh74G0bzI/08cG2SN82+lyTpma7PEcS/BV5S\nVTtgNAkf8JeMHuojSVqg+lzFtM9MODR399xPkvQM1ucI4i+SfB74RFv/BeDy4UqSJE2DPlcx/XqS\nU4F/wOipbxuq6tLBK5MkTVSvq5iq6lPApwauRZI0RRxLkCR1MiAkSZ12GxBJNrX3D85fOZKkaTHb\nGMQRSV4JnJzkIkYD1I+rqq8MWpkkaaJmC4j3AGcDy4Hf32VbAScOVZQkafJ2GxBVdQlwSZJ/V1Xv\nm8eaJElToM99EO9LcjLwitZ0VVV9dtiyJEmT1meyvt8BzgBuaa8zWpskaQHrc6PcG4Bjq+oxgCQX\nMJry+91DFiZJmqy+90EsHVs+aIhCJEnTpc8RxO8AX01yJaNLXV+BRw+StOD1GaT+RJKrgJcwCoiz\nqur/DF2YJGmy+k7W9x1g48C1SJKmiHMxSZI6GRCSpE6zBkSSfZLcNF/FSJKmx6wB0e59+FqSFfNU\njyRpSvQ5xXQEcHOSTUk2zrzm2inJUUmuTHJrkpuTnNHaD0lyRZLb2vvBrT1JPpZkS5Ibkxz31P40\nSdJT0ecqpt/ay89+FPi1qvpKkgOB65NcAfwLYFNVfSDJ2YxmjD0LOAlY1V5/Hzi3vUuSJmDOI4iq\nuhq4A9ivLV8HzPksiKr6zswzI6rqQeBW4EhgDXBB63YBcEpbXgNcWCPXAEuTHLFnf44k6enSZ7K+\nfwlcAvyX1nQk8Ok9+ZIkK4EXAdcCh7f7Kmbur3ju2OduHdttW2uTJE1AnzGItwEvBx4AqKrbeOJH\nfU5JngP8D+BdVfXAbF072qrj89Yn2Zxk886dO/uWIUnaQ30C4pGq+sHMSpIldPxwd0myH6Nw+JOq\n+lRrvmvm1FF739HatwFHje2+HNi+62dW1YaqWl1Vq5ctW9anDEnSXugTEFcn+Q3gx5L8Q+CTwGfm\n2ilJgPOAW6tq/JGlG4G1bXktcNlY++ntaqYTgPtnTkVJkuZfn6uYzgbWAV8HfhW4HPjjHvu9HPhl\n4OtJbmhtvwF8ALg4yTrgTuC0tu1y4PXAFuBh4C09/wZJ0gD6zOb6WHtI0LWMTi19s6rmPMVUVX9F\n97gCwGs6+hej8Q5J0hSYMyCSvAH4I+BbjH7wj07yq1X150MXJ0manD6nmD4MvLqqtgAk+bvA5wAD\nQpIWsD6D1DtmwqG5nSeuPJIkLVC7PYJIcmpbvDnJ5cDFjMYgTmN0N7UkaQGb7RTTPx1bvgt4ZVve\nCRw8WEWSpKmw24CoKi8zlaRFrM9VTEcD7wBWjvevqpOHK0uSNGl9rmL6NKM7oj8DPDZsOZKkadEn\nIL5fVR8bvBJJ0lTpExD/Mck5wBeAR2YaZ571IElamPoExPMZzal0Ik+cYqq2LklaoPoExBuB541P\n+S1JWvj63En9NWDp0IVIkqZLnyOIw4FvJLmOHx2D8DJXSVrA+gTEOYNXIUmaOn2eB3H1fBQiSZou\nfe6kfpAnnkG9P7Af8L2q+vEhC5MkTVafI4gDx9eTnAIcP1hFkqSp0Ocqph9RVZ/GeyAkacHrc4rp\n1LHVfYDVPHHKSZK0QPW5imn8uRCPAncAawapRpI0NfqMQfhcCElahGZ75Oh7Ztmvqup9A9QjSZoS\nsx1BfK+j7QBgHXAoYEBI0gI22yNHPzyznORA4AzgLcBFwId3t58kaWGYdQwiySHAmcCbgQuA46rq\n3vkoTJI0WbONQfwecCqwAXh+VT00b1VJkiZuthvlfg34CeA3ge1JHmivB5M8MD/lSZImZbYxiD2+\ny1qStHAYApKkTgaEJKmTASFJ6mRASJI6DRYQSc5PsiPJTWNthyS5Islt7f3g1p4kH0uyJcmNSY4b\nqi5JUj9DHkF8HHjdLm1nA5uqahWwqa0DnASsaq/1wLkD1iVJ6mGwgKiqLwH37NK8htEd2bT3U8ba\nL6yRa4ClSY4YqjZJ0tzmewzi8Kr6DkB7f25rPxLYOtZvW2t7kiTrk2xOsnnnzp2DFitJi9m0DFKn\no63zqXVVtaGqVlfV6mXLlg1cliQtXvMdEHfNnDpq7zta+zbgqLF+y4Ht81ybJGnMfAfERmBtW14L\nXDbWfnq7mukE4P6ZU1GSpMno80zqvZLkE8CrgMOSbAPOAT4AXJxkHXAncFrrfjnwemAL8DCj505I\nkiZosICoql/azabXdPQt4G1D1SJJ2nPTMkgtSZoyBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ\n6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ\n6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ\n6mRASJI6TVVAJHldkm8m2ZLk7EnXI0mL2dQERJJ9gf8MnAQcA/xSkmMmW5UkLV5TExDA8cCWqrq9\nqn4AXASsmXBNkrRoTVNAHAlsHVvf1tokSROwZNIFjElHWz2pU7IeWN9WH0ryzUGrWlwOA7476SKm\nQT60dtIl6Ef5b3PGOV0/lXvsJ/t0mqaA2AYcNba+HNi+a6eq2gBsmK+iFpMkm6tq9aTrkHblv83J\nmKZTTNcBq5IcnWR/4BeBjROuSZIWrak5gqiqR5O8Hfg8sC9wflXdPOGyJGnRmpqAAKiqy4HLJ13H\nIuapO00r/21OQKqeNA4sSdJUjUFIkqaIASGnONHUSnJ+kh1Jbpp0LYuRAbHIOcWJptzHgddNuojF\nyoCQU5xoalXVl4B7Jl3HYmVAyClOJHUyINRrihNJi48BoV5TnEhafAwIOcWJpE4GxCJXVY8CM1Oc\n3Apc7BQnmhZJPgH8L+BnkmxLsm7SNS0m3kktSerkEYQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASHt\nRpKlSf7VPHzPq5K8bOjvkfaUASHt3lKgd0BkZG/+m3oVYEBo6ngfhLQbSWZmtv0mcCXwAuBgYD/g\nN6vqsiQrgT9v218KnAK8FjiL0ZQltwGPVNXbkywD/ghY0b7iXcDfAtcAPwR2Au+oqv85H3+fNBcD\nQtqN9uP/2ar6e0mWAH+nqh5IchijH/VVwE8CtwMvq6prkvwE8NfAccCDwBeBr7WA+FPgD6vqr5Ks\nAD5fVT+b5L3AQ1X1ofn+G6XZLJl0AdIzRID3J3kF8BijKdEPb9u+XVXXtOXjgaur6h6AJJ8Efrpt\ney1wTPL4BLo/nuTA+She2hsGhNTPm4FlwIur6v8luQN4dtv2vbF+XdOnz9gHeGlV/d/xxrHAkKaK\ng9TS7j0IzPwf/kHAjhYOr2Z0aqnL3wCvTHJwOy31z8a2fYHRxIgAJDm243ukqWFASLtRVXcDX05y\nE3AssDrJZkZHE9/YzT5/C7wfuBb4S+AW4P62+Z3tM25Mcgvw1tb+GeCNSW5I8nOD/UHSHnKQWnqa\nJXlOVT3UjiAuBc6vqksnXZe0pzyCkJ5+701yA3AT8L+BT0+4HmmveAQhSerkEYQkqZMBIUnqZEBI\nkjoZEJKkTgaEJKmTASFJ6vT/AUeMG8jkWPD8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xed5d2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Target 分布，看看各类样本是否均衡\n",
    "sns.countplot(data.Outcome);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.84488505  1.63075243  0.25303625 ...,  0.61015436 -1.15935199\n",
      "   0.40494237]\n",
      " [ 0.3429808  -0.49745345  0.14964075 ...,  0.62284628 -0.94492356\n",
      "  -0.44593516]\n",
      " [-0.25095213  2.19410104 -0.05715025 ..., -0.13866919 -0.52210695\n",
      "   0.06459135]\n",
      " ..., \n",
      " [ 0.04601433 -0.84172205 -0.2122435  ..., -0.92556851 -0.97814487\n",
      "  -1.04154944]\n",
      " [ 2.12477957 -1.12339636  0.25303625 ..., -0.24020459 -0.51908683\n",
      "   0.14967911]\n",
      " [ 0.3429808   0.47275805  0.66661825 ..., -4.06047387  0.50775352\n",
      "   3.04266271]]\n"
     ]
    }
   ],
   "source": [
    "#得到目标值\n",
    "\n",
    "y_train = data['Outcome']\n",
    "\n",
    "#得到训练特征\n",
    "x_train = data.drop('Outcome',axis=1)\n",
    "\n",
    "x_train = np.array(x_train)\n",
    "\n",
    "\n",
    "#标准化处理\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "x_train = ss_X.fit_transform(x_train)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#80%用来巡礼阿宝\n",
    "train_X,test_X, train_y, test_y = train_test_split(x_train,  \n",
    "                                                   y_train,  \n",
    "                                                   test_size = 0.8,  \n",
    "                                                   random_state = 0)  \n",
    "\n",
    "\n",
    "print (train_X)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#缺省的logisticRegression 参数\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('logloss of each fold is:%', array([ 0.58978303,  0.61172808,  0.80843559,  0.36466707,  0.50189047,\n",
      "        0.43902988,  0.55211238,  0.61505575,  0.37422983,  0.51528353]))\n",
      "('cv logloss is:%', 0.53722156158573786)\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优(模型选择)\n",
    "#分类任务中交叉验证缺省是采用StratifiedFFold\n",
    "\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "loss = cross_val_score(lr,train_X,train_y,cv=10,scoring='neg_log_loss')\n",
    "\n",
    "print('logloss of each fold is:%',-loss)\n",
    "\n",
    "print('cv logloss is:%',-loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#参数调优\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys=['l1','l2']\n",
    "\n",
    "Cs=[0.001,0.01,0.1,1,10,100,1000,10000]\n",
    "\n",
    "tuned_parameters = dict(penalty=penaltys,C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "\n",
    "grid = GridSearchCV(lr_penalty,tuned_parameters,cv=5,scoring='neg_log_loss')\n",
    "\n",
    "grid.fit(train_X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "J:\\MyInstall\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 0.00219994,  0.00220003,  0.00159998,  0.00180001,  0.00180001,\n",
       "         0.0019999 ,  0.002     ,  0.00180006,  0.00199995,  0.00199995,\n",
       "         0.00220008,  0.00219998,  0.00419998,  0.00319996,  0.00160003,\n",
       "         0.00200005]),\n",
       " 'mean_score_time': array([ 0.004     ,  0.00319996,  0.0026    ,  0.002     ,  0.00259995,\n",
       "         0.00280004,  0.00219998,  0.00240002,  0.00280004,  0.00220008,\n",
       "         0.00239992,  0.00559998,  0.00660005,  0.00380001,  0.00219998,\n",
       "         0.00239992]),\n",
       " 'mean_test_score': array([-0.69314718, -0.67909111, -0.69314718, -0.6073056 , -0.55806209,\n",
       "        -0.5287282 , -0.54844042, -0.55935351, -0.57853167, -0.5797249 ,\n",
       "        -0.58246763, -0.58259289, -0.5828696 , -0.58289142, -0.58291763,\n",
       "        -0.5829214 ]),\n",
       " 'mean_train_score': array([-0.69314718, -0.67766716, -0.69314718, -0.5966194 , -0.54071143,\n",
       "        -0.47951115, -0.45151563, -0.44892888, -0.44745698, -0.44742769,\n",
       "        -0.44740618, -0.44740589, -0.44740566, -0.44740565, -0.44740565,\n",
       "        -0.44740565]),\n",
       " 'param_C': masked_array(data = [0.001 0.001 0.01 0.01 0.1 0.1 1 1 10 10 100 100 1000 1000 10000 10000],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_penalty': masked_array(data = ['l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1'\n",
       "  'l2'],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'},\n",
       "  {'C': 10000, 'penalty': 'l1'},\n",
       "  {'C': 10000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([15, 14, 15, 13,  3,  1,  2,  4,  5,  6,  7,  8,  9, 10, 11, 12]),\n",
       " 'split0_test_score': array([-0.69314718, -0.68122237, -0.69314718, -0.62407106, -0.57592897,\n",
       "        -0.57600438, -0.60306955, -0.62320614, -0.65079767, -0.65308667,\n",
       "        -0.65724205, -0.65747315, -0.65789634, -0.65793249, -0.65799285,\n",
       "        -0.65797864]),\n",
       " 'split0_train_score': array([-0.69314718, -0.67655621, -0.69314718, -0.58951195, -0.53557135,\n",
       "        -0.46722067, -0.43932579, -0.43578433, -0.43410461, -0.43406841,\n",
       "        -0.43404228, -0.43404191, -0.43404163, -0.43404162, -0.43404161,\n",
       "        -0.43404161]),\n",
       " 'split1_test_score': array([-0.69314718, -0.68099636, -0.69314718, -0.62006842, -0.55061908,\n",
       "        -0.5671186 , -0.60074226, -0.62338804, -0.64615027, -0.64838829,\n",
       "        -0.65142383, -0.65166731, -0.65197274, -0.65200561, -0.65203176,\n",
       "        -0.65203955]),\n",
       " 'split1_train_score': array([-0.69314718, -0.67641722, -0.69314718, -0.59084106, -0.54485702,\n",
       "        -0.47201948, -0.44500842, -0.44348904, -0.44232126, -0.44229743,\n",
       "        -0.4422815 , -0.44228126, -0.4422811 , -0.4422811 , -0.44228109,\n",
       "        -0.44228109]),\n",
       " 'split2_test_score': array([-0.69314718, -0.67762026, -0.69314718, -0.5960125 , -0.54130024,\n",
       "        -0.48847305, -0.48023313, -0.48597717, -0.49535902, -0.49599931,\n",
       "        -0.49747971, -0.4975406 , -0.49769452, -0.49770242, -0.49771341,\n",
       "        -0.49771869]),\n",
       " 'split2_train_score': array([-0.69314718, -0.67858833, -0.69314718, -0.60149659, -0.5445489 ,\n",
       "        -0.48910342, -0.46319352, -0.46066139, -0.45932311, -0.45929598,\n",
       "        -0.4592765 , -0.45927624, -0.45927602, -0.45927602, -0.45927601,\n",
       "        -0.45927601]),\n",
       " 'split3_test_score': array([-0.69314718, -0.6785873 , -0.69314718, -0.60522176, -0.58757904,\n",
       "        -0.5382414 , -0.58968844, -0.58711603, -0.6124943 , -0.6122243 ,\n",
       "        -0.61568621, -0.61565459, -0.61599029, -0.61601044, -0.61602881,\n",
       "        -0.61604616]),\n",
       " 'split3_train_score': array([-0.69314718, -0.67811108, -0.69314718, -0.59685571, -0.53032616,\n",
       "        -0.47111643, -0.43702187, -0.43487083, -0.43297389, -0.43294947,\n",
       "        -0.43292136, -0.43292112, -0.43292082, -0.43292081, -0.43292081,\n",
       "        -0.43292081]),\n",
       " 'split4_test_score': array([-0.69314718, -0.67688703, -0.69314718, -0.59004038, -0.53328455,\n",
       "        -0.47001564, -0.46295342, -0.47140616, -0.48160365, -0.48266391,\n",
       "        -0.4841642 , -0.48428644, -0.48444408, -0.48445543, -0.48446985,\n",
       "        -0.4844724 ]),\n",
       " 'split4_train_score': array([-0.69314718, -0.67866295, -0.69314718, -0.60439167, -0.54825371,\n",
       "        -0.49809574, -0.47302858, -0.46983879, -0.46856201, -0.46852715,\n",
       "        -0.46850928, -0.4685089 , -0.46850872, -0.46850871, -0.46850871,\n",
       "        -0.46850871]),\n",
       " 'std_fit_time': array([  4.00042545e-04,   7.48353705e-04,   4.89842988e-04,\n",
       "          7.48379193e-04,   4.00042545e-04,   9.53674316e-08,\n",
       "          8.94468989e-04,   4.00066376e-04,   8.94468989e-04,\n",
       "          6.32485093e-04,   7.48277239e-04,   4.00018706e-04,\n",
       "          7.48328219e-04,   7.48417431e-04,   4.89784582e-04,\n",
       "          6.32409699e-04]),\n",
       " 'std_score_time': array([  1.54917339e-03,   3.99994861e-04,   4.89920847e-04,\n",
       "          1.16800773e-07,   7.99953946e-04,   7.48277239e-04,\n",
       "          4.00018706e-04,   8.00025464e-04,   1.16610585e-03,\n",
       "          7.48277239e-04,   4.89979242e-04,   4.80005145e-03,\n",
       "          3.32267278e-03,   7.48417431e-04,   4.00018706e-04,\n",
       "          4.89979242e-04]),\n",
       " 'std_test_score': array([  9.99482874e-17,   1.75110560e-03,   9.99482874e-17,\n",
       "          1.31596022e-02,   2.06330689e-02,   4.18511804e-02,\n",
       "          6.18986103e-02,   6.61060991e-02,   7.34889597e-02,\n",
       "          7.39303208e-02,   7.49332477e-02,   7.49782027e-02,\n",
       "          7.50762956e-02,   7.50869066e-02,   7.50995431e-02,\n",
       "          7.50978181e-02]),\n",
       " 'std_train_score': array([ 0.        ,  0.00098324,  0.        ,  0.00579924,  0.00667941,\n",
       "         0.01195869,  0.01414344,  0.01396335,  0.01414672,  0.01414533,\n",
       "         0.01414858,  0.01414856,  0.01414859,  0.01414859,  0.0141486 ,\n",
       "         0.0141486 ])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.528728195023\n",
      "{'penalty': 'l2', 'C': 0.1}\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3Xd8VMXawPHfpFfSEwIhhCC9hCZF\nEKQoSBNEsAL6iopiV4qIXmxcEFEsV7pUucBVUQRFiiJIUzqICkgJPYUkpG+yO+8fu4kBEtgku9mU\n53vvfnb37JwzzwrkyZxz5hmltUYIIYQoLSdHByCEEKJykIQihBDCJiShCCGEsAlJKEIIIWxCEooQ\nQgibkIQihBDCJiShCCGEsAlJKEIIIWxCEooQQgibcHF0AGUpODhYR0VFOToMIYSoUHbv3p2gtQ65\nUbsqlVCioqLYtWuXo8MQQogKRSl1ypp2cspLCCGETUhCEUIIYROSUIQQQthElbqGUpicnBzOnDlD\nVlaWo0Mptzw8PIiIiMDV1dXRoQghyrEqn1DOnDmDr68vUVFRKKUcHU65o7UmMTGRM2fOUKdOHUeH\nI4Qox6r8Ka+srCyCgoIkmRRBKUVQUJCM4IQQN1TlEwogyeQG5L+PEMIaklBK4N5Z27l31nZHhyGE\nEOWKJJRywMfHJ/91r1698Pf3p2/fvoW2HTVqFC1atKBx48Z4enrSokULWrRowRdffFGsPvfs2cPa\ntWtLFbco307d3pJTt7d0dBhWqUixQsWKtyxjrfIX5cub0aNHk5GRwaxZswr9/D//+Q8AJ0+epG/f\nvuzbt69E/ezZs4dDhw7Rq1evEscqhBAFyQilnOnevTu+vr4l2vfo0aP07NmT1q1b07lzZ44cOQLA\nsmXLaNq0KTExMXTt2pXMzEzefPNNPv/88xKNbqqqU0OHcWroMEeHIUS5JSOUAt749ncOn7t8w3aH\nz5vbWHMdpXGNavyrX5NSx2aNxx9/nLlz51K3bl22bt3K008/zbp163jjjTfYtGkTYWFhJCcn4+np\nyeuvv86hQ4eYPn16mcRWKVw46OgIhCjXJKFUEsnJyezYsYNBgwblb8vNzQWgY8eODBs2jMGDB3P3\n3Xc7KkQhRCUnCaUAa0cSeSOT5U90sGc4xaK1Jjg4uNBrKnPmzGHnzp2sXr2amJgYDhw44IAIhRCV\nnVxDqSQCAgIIDw9n5cqVAJhMJvbv3w/A8ePHad++PW+99RYBAQGcPXsWX19fUlNTHRmyEKKSkYRS\nztx6660MHjyYjRs3EhERwQ8//GD1vsuWLWPmzJnExMTQpEkTVq9eDcALL7xAs2bNaNasGT169KBp\n06Z069aN/fv307JlS7koL4SwCTnlVQ6kpaXlv96yZYtV+0RFRXHo0KErtkVHRxeagFatWnXNtpCQ\nEFlsTAhhUw5JKEqpQGA5EAWcBIZorZOKaFsN+ANYqbV+2rJtExAOZFqa3aG1jrNv1P8oT9dOhBCi\nvHDUCGUcsFFrPVkpNc7yfmwRbd8Cfi5k+4Naa/kVW1Qo2mhE5+SgDQbzo8Brk8EAOTmYDAa0IQed\nY3m+pm02OisTnXEZnZmKKSMVnZWBzko3bzdkobOzMGWkgobYO9sUHY/WVgR9ww1WHOP6+5jSzdfz\nTt1RMWafV6R482LN+XM3rg1b27UvRyWUu4DbLK8XApsoJKEopVoDYcBaoOh/FUKUAZPBRG66kXPj\nX70mGVz52pwITHkJocBnGI02jUk5aZSzNj875b0H5eIMJkCBKdtwo6NYtenGwVx/p0I/zdsnL9+U\nIFc5REWKNz9Wk927clRCCdNanwfQWp9XSoVe3UAp5QRMA4YC3Qs5xnyllBH4EnhbW/WrlhDFl3P2\nLPEffUzWRQMoSN+2DeXqinJzMz9cXVEuTji5KpSHO0q5mn+wKyNK5aLIMT+0AaWzUKYslCkTZcpA\nkYtTflIomCBAubqhvKqhvKuhvP1QXn4o30CcfINQPoHgE4TyCgTPAPDMew4AV09QKr9+U+31ex38\nX/DGKlKsULHizYvVtdHNdu/LbglFKbUBqF7IR69aeYingO+01qcLKZ/+oNb6rFLKF3NCGQosKiKO\nx4HHASIjI63sWgjITUoiceZMkpb+F5TC1c+JoCaXCegRBplJkBkHmZfMr025RR/I1avAD/3gf37w\n5z28Aq/dlpcYhKhA7JZQtNY9ivpMKXVRKRVuGZ2EA4VdUO8A3KqUegrwAdyUUmla63Fa67OWPlKV\nUkuBthSRULTWs4HZAG3atLHNKGZ+H/PzI2tscjhRvpjS00lcuJBL8z7DlJmJ38ABhDz9NIa32+AV\nlAaXz5h/4Ic2vHJkUFiS8PAHVw9HfyUhyoSjTnmtAoYDky3P31zdQGv9YN5rpdTDQBut9TillAvg\nr7VOUEq5An2BDWUStZ34+Pjk3zrcq1cvduzYQadOnfLnkRQ0atQotm7disFg4MSJEzRo0ACACRMm\ncM8991jV38qVKzl27BijR4+23ZeoBLTBQNL//kfCjJkYExLw6dGd0BdewL1uXTiyDteQNJJjA/B/\n8xdHhypEueSohDIZWKGUehSIBQYDKKXaACO11iOus6878IMlmThjTiZz7BxvmbFV+frc3FxcXAr/\n4x04cKBtgq0ktMnE5e++J/7DD8k5fRqvm28m9JOP8WzRwtwgOxVWv4Ahw5WUM/74OzZcIcothyQU\nrXUihVxot9wGfE0y0VovABZYXqcD9r33zYG6d+/Opk2bSrRvp06d6NKlC1u2bOHuu++mTp06TJo0\nCYPBQEhICEuWLCE0NJS5c+fmVxp+6KGHCAoK4rfffuPChQtMmzatyiQcrTXpv2wl7oP3yT78B+4N\nGlBr9iy8b731ymWPN74Fl8+SeCwctCyHLERRZKZ8Qd+Ps65E+QVLccW8aynXU70Z3Dm5dHEVw+XL\nl9m8eTMASUlJ9O/fH6UUM2fOZNq0aUyZMuWafeLi4ti6dSsHDx5kyJAhVSKhZB44QNy098nYuRPX\niAhqTH2Xan36oJyuqkZ0+lf4dTa0fQzD1gp9ZlUIu5OEUsncd999+a9jY2MZMmQIFy5cIDs7m/r1\n6xe6z4ABA1BK0bx5c86ePVtWoTpE9vHjxE//kNR163AODCTs1VcJuHcIys3t2sa5Blj1DFSrAd1f\nhw8koQhxPZJQCrJ2JFGO7/Ly9vbOfz1q1CjGjx9P79692bBhA5MnF/793N3d819X1uk8ORcvkvDJ\nJyR/tRInd3eCn36awIcfxtnHu+idfvkA4v+E+5eDe8lW0RSiKpGEUomlpKRQs2ZNtNYsXLjQ0eE4\nhDElhcQ5c7i0eAnaZCLggQcIHvkELkFB198x7k/YPBWaDoIGvcomWCEqOEko5cytt97Kn3/+SVpa\nGhEREcybN4+ePXuW6FgTJ05k4MCBRERE0LZtW86fP2/jaMsvU1YWlxYvJnHOXEypqVTr15eQZ5/F\nLSLCip1N8O2z4O4Dva695lRRVIRZ3HkqUqxQseIty1hVZT3FUZg2bdroq0u2//HHHzRq1Kh4ByrH\np7zspUT/nRxA5+aS/NVXJHzyH3Lj4vDu0pnQF1/EwzJfxyq/zoHvXoYBM6DFA/9sr4J/7kIAKKV2\na61vWE9RRiglIT9Qyh2tNanr1hM/fTqGEyfwjImh5rT38Lq5mPWLUs7AhjcguivE3G+fYIWopCSh\niAovfcdO4t5/n6wDB3CrW5eITz7Gp3t3CqkBd31aw5qXQBuh3/Rrq+fKLxJCXJckFOFQp4YOA6D2\n4kJLsV1X1uHDxL3/Aem//IJL9eqEv/MOfgPuQjk7lyyY31fCkbVwxzsQEFWyYwhRhUlCERWOITaW\n+A8/4vKaNTj7+RE6ZgwBD9yPk0cpijBmXILvx0CNltBupO2CFaIKkYQiKozc+HgSZswkacUKlIsL\nQU88QdCj/4dztWqlP/i618xJZehKcJZ/FkKUhPzLKYFH1j4CwPxe8x0cSdVgTEsjcd48Li1chM7O\nxn/wPQQ/9RSuodesy1Yyf/8E+5ZApxfNpXKEECXidOMmwt58fHwA2LdvHx06dKBJkyY0b96c5cuX\nX9N21KhRtGjRgsaNG+Pp6UmLFi1o0aIFX3zxRbH63LNnD2vXrrVJ/PZiMhhIXLCAv3vcTuKMmfh0\n6UzdNasJnzjRdsnEkAGrn4fAutBljG2OKUQVJSOUcsTLy4tFixZRr149zp07R+vWrenZsyf+/v8U\nTLe2fP2N7Nmzh0OHDtGrV/mbBa6NRlJWfUv8xx+Re+483rd0IOTFl/Bs2sT2nW2aBEkn4eE1skKi\nEKUkI5RypH79+tSrVw+AGjVqEBoaSnx8vNX7Hz16lJ49e9K6dWs6d+7MkSNHAFi2bBlNmzYlJiaG\nrl27kpmZyZtvvsnnn39eotGNvWitSf3xJ04MGMj5V17BJSCQyM/mEfnZZ/ZJJuf2wvb/QKvhENXJ\n9scXooqREUoBU36dwp+X/rxhu7w2eddSrqdhYEPGth1b7Fh+/fVXDAYDdevWtXqfxx9/nLlz51K3\nbl22bt3K008/zbp163jjjTfYtGkTYWFhJCcn4+npyeuvv56/Jkp5kLFnD3HvTSNzzx5ca0dS84P3\n8e3Z89py8rZizDFXEvYOgdvftE8fQlQxklDKofPnzzN06FAWLlyIk5U/UJOTk9mxYweDBg3K35ab\nmwtAx44dGTZsGIMHD+buu++2S8wlZcrKIuf0aU498CDOIcFUn/gv/AcNQrm62rfj7Z+Y174Zshg8\nZQ1GIWxBEkoB1o4k7HmX1+XLl+nTpw9vv/027du3t3o/rTXBwcGFXlOZM2cOO3fuZPXq1cTExHDg\nwAFbhlxiOjeX7CNH0Lm5hDz/PIHDhuLk5WX/jhP/hk2ToWFfaNzf/v0JUUXINZRyxGAwMHDgwPzR\nRHEEBAQQHh7OypUrATCZTOzfvx+A48eP0759e9566y0CAgI4e/Ysvr6+pKam2vw7FMfl775DZ2fj\nVqcOwSOfKJtkojV8+xw4u0Hv9+zfnxBViCSUcmTFihVs3ryZBQsW5N8OXJy7uJYtW8bMmTOJiYmh\nSZMmrF69GoAXXniBZs2a0axZM3r06EHTpk3p1q0b+/fvp2XLlg65KK+NRhJmzkJ5euLsX4annPYu\nhpNbzNdNqoWXXb9CVAFyyqscSEtLA+Chhx7ioYcesmqfqKgoDh06dMW26Ohofvjhh2varlq16ppt\nISEhXF3KvyylrluH4fhx3OrWLX4RxxJ3egHWTYDaHc13dgkhbEoSSgnIDPnS0SYTCTNm4hYdjXNA\nQNl1/P0YyMmCfh+Cve4eE6IKk39Vosyl/fgj2UeOEDzyibIbnfyxGg5/Y54NH1yvbPoUooqRhCLK\nlNaahE9n4BoZSbXevfnz0p9Wzf0plawU8wqMYU2h43MlPsy9s7Zz76ztNgxMiMpFEoooU+mbN5N1\n+DDBTzyOcimjM64bJkLaRej/ETjbeX6LEFWYJBRRZvJHJzVq4Ne/jOZ/nNoGuz6D9k9BzdZl06cQ\nVZQklBI4NXRY/kqDwnoZ27eTuX8/QY8/Zv+Z8GC+AL/qWfCPhK7j7d+fEFWcJJRyoKzL169cuZKp\nU6faLH5rJcyYiUtoKH4Fyr8se6YJy56xQ+FHgM1TIfEo9J0Obt726UMIkU9uGy5HbFm+Pjc3F5ci\nrlEMHDjQ9sHfQMauXWT89hth41/Byc3N/h1eOARbp0PM/XBTd5sc8qRb3sz6L21yPCEqG4ckFKVU\nILAciAJOAkO01kmFtDMCBy1vY7XW/S3b6wDLgEBgDzBUa22wf+T2Vb9+/fzXBcvX+1s5k7xTp050\n6dKFLVu2cPfdd1OnTh0mTZqEwWAgJCSEJUuWEBoayty5c/MrDT/00EMEBQXx22+/ceHCBaZNm2aX\nhJPw6Qycg4LwL2ZJmRIxGc2VhD38oeck+/cnhAAcN0IZB2zUWk9WSo2zvC+sMmOm1rpFIdunAB9o\nrZcppWYCjwIzShvUhUmTyP7jxrewZv1pbmPNdRT3Rg2pPr745+9LUr4ezMUlN2/eDEBSUhL9+/dH\nKcXMmTOZNm0aU6ZMuWafuLg4tm7dysGDBxkyZIjNE0rm/v2kb9tG6OiXcfIsg0Wsds6Cc3tg0Dzw\nCrR/f0IIwHEJ5S7gNsvrhcAmCk8o11DmmXDdgAcK7D8RGySU8qIk5evz3HffffmvY2NjGTJkCBcu\nXCA7O/uKEVBBAwYMQClF8+bNOXv2bKliL0zCpzNw9vcnoEBsdpN0Cn58C+rdAU0H3bi9EMJmHJVQ\nwrTW5wG01ueVUkUtEO6hlNoF5AKTtdZfA0FAstY619LmDFDTFkFZO5LIG5nUXrzIFt1eoaTl6/N4\ne/9z8XnUqFGMHz+e3r17s2HDBiZPnlzoPu7u7vmvtdbFD/o6Mn//nbSffybk+edw8rbzhXGtYfUL\ngII+70NZzcIXQgB2TChKqQ1A9UI+erUYh4nUWp9TSkUDPyqlDgKXC2lX5E9BpdTjwOMAkZGRxei6\n7JWmfH1hUlJSqFmzJlprFi5caIMIiy9x5kycqlUj4MEH7d/ZgRXw90a4813wr2X//oQQV7DbbcNa\n6x5a66aFPL4BLiqlwgEsz3FFHOOc5fk45tNiLYEEwF8plZcMI4Bz14ljtta6jda6TUhIiM2+nz2U\ntnz91SZOnMjAgQPp0qULYWFhNozUOll/HSF1/QYCH3oIZ19f+3aWngBrx0HEzXDzCPv2JYQolKNO\nea0ChgOTLc/fXN1AKRUAZGits5VSwUBH4F2ttVZK/QTcg/lOr0L3r0hsVb7+l19+ueL9oEGDrlgS\nOM+IEf/8wF2yZEmhsdhC4qyZOHl5EThsqM2OWaS1r0B2KvT/GJyc7d+fEOIajkook4EVSqlHgVhg\nMIBSqg0wUms9AmgEzFJKmTCPpCZrrQ9b9h8LLFNKvQ3sBeaVZfD2uHZS2WQfP87l79cSNGKE/RfQ\nOroeDq6ALmMhtJF9+xJCFMkhCUVrnQhcM9tMa70LGGF5vQ1oVsT+x4G29oxRlE7irFkoDw8CH3nY\nvh1lp5kvxAc3gFtfsm9fFUy7+ebR6c5HZCKmKBsyUx7znU1lti5HBVTcO78MsbGkrF5D4NChuATa\neR7Ij29Dyhn4vx/Axf3G7YUQdlPla3l5eHiQmJho89tlKwutNYmJiXh4eFi9T8Ls2ShnZwL/7xE7\nRgac2QU7Z5ovwke2s29fwq7azR+UP6KqCCpSvGUZa5UfoURERHDmzBni4+MdHUq55eHhQUREhFVt\nc86eJeXrbwi4915cQ4uaXmQDuQZzeZVqNaD76/brRwhhtSqfUFxdXalTp46jw6g0EubOBaUIGvGo\nfTva+iHEHYb7l4NHNfv2JYSwSpU/5SVsJ+fiRVK++BL/gQNxDQ+3X0fxf8Hmd6HJ3dCgl/36EUIU\niyQUYTOJ8+ahTSaCHn/Mfp2YTOZFs1y94M5rC10KIRynyp/yEraRm5BA8vIV+PXvj5uV11tKZPdn\ncHoH3PUp+NjxGo0QothkhCJsInH+fHRODsFPPG6/TlLOwvqJUKcLtHjghs2FEGVLEoootdykJJL+\nu4xqvXvjFhVln060hjUvgSkX+k2XSsJClEOSUESpXVq4EJ2ZSfDIJ+zXyeGv4cj30HU8BEbbrx8h\nRIlJQhGlYrx8maQln+N7xx2433STfTrJuATfjYbwGGj/lH36EEKUmlyUF6VyafFiTGlpBD850n6d\nrH/NnFQe+hKc5a+sEOWVjFBEiRnT0ri0aDE+3brh0bChfTo5vgn2LoFbnjGPUIQQ5ZYkFFFiSUv/\niyklxX6jE0MGfPs8BNSB28bZpw8hhM3I+QNRIqaMDC7Nn493p054Nit0lYHS+3kyJJ2A4d+Cq6d9\n+hBC2IyMUESJJC1fgTEpieCnnrRPB+f2wbZPoNUwqNPZPn0IIWxKEoooNlNWFomfzcOrXTu8WrUq\n8XG01py6fIqjSUc5nHj4nw+MueZKwt7BcPubNohYCFEWrEooSqmOSilvy+uHlFLvK6Vq2zc0UV4l\nf/ElxvgEgp8s3ejkq6NfEZ8Zz2XDZe5dfS+jfx5N7OVY2PEfuHAAek8FzwAbRS2EsDdrr6HMAGKU\nUjHAGMxruC8CutgrMFE+mQwGEufOxbNVK7zalXwV5uPJx5n862R83Xyp61eXtuFtWXx4MRtOrWdQ\naiojG/QkuFF/G0YuhLA3a0955WrzkoZ3AR9qrT8EfO0XliivUlZ+Te6FCwQ/+WSJl03ONmYzevNo\nvFy9qFOtDi5OLjzT8hm+G7iGQSZPvvT2pLfxOB/t/ZhUQ6qNv4EQwl6sTSipSqlXgIeANUopZ8DV\nfmGJ8kjn5JA4ezYezZrh3aljiY8zbdc0jiQd4a2Ob+Hm7Ja/PfjPtUw4eZhv6v0fXWp1Zc7BOfT+\nqjeLfl9EtjHbFl9BCGFH1iaUe4Fs4FGt9QWgJjDVblGJcinl29XknD1bqtHJT7E/8d8//8vQxkPp\nHNEZLhw0P1IvwrpXIfIWIjs8z9QuU1nWdxmNAhsxdddU+q3sxzfHvsFoMtr4WwkhbMXqEQrmU11b\nlFL1gRbAf+0XlihvtNFI4qxZuDdqhE/X20p0jIvpF3lt22s0CmzE862ev/LD78dATib0/wiczH8t\nmwQ1YfYds5lzxxwCPQKZsHUC93x7Dz+f/hnzGVghRHlibULZDLgrpWoCG4FHgAX2CkqUP5e/+x7D\nqVMEjxxZotGJ0WRk3JZxGIwG3u38bv6prnRDLo1SUszVhLuMgeB61+zbPrw9/+3zX97r8h45phye\n/vFpHl77MPvi9pX6ewkhbMfahKK01hnA3cDHWuuBQBP7hSXKE20ykTBrJu71bsL39h4lOsbcg3PZ\ndXEXr7Z7lSi/qPztniYTI1ISIbQJ3PJckfsrpegZ1ZOVd63ktfavcTr1NEO/H8ozPz7DsaRjJYpJ\nCGFbVicUpVQH4EFgjWWbs31CEuVN6voNGI79TdATI1FOxZ8LuzduLzP2z6B3nd70r3vlrcD3pibh\nbzJC/4/Bxa2II/zD1cmVIQ2GsHrgap5t+Sy7Luxi0LeDmPDLBM6nnS92bEII27H2p8PzwCvASq31\n70qpaOAn+4UlygutNQkzZ+IWFUW1O3sVe//LhsuM3TyWcO9wXmv/2pWny7LT6JaRys+ePhDRuljH\n9XL14rHmj/H93d8ztNFQvj/xPX1X9mXqb1NJzkoudpxCiNKzKqForX/WWvcHPlVK+Witj2utn7Vz\nbKIcSPtpE9l//EHQE0+gnIs3KNVaM3HbROIz4nm387v4uPlc2eCPVXhozY9eJZ/S5O/hz8s3v8zq\ngavpHd2bJX8s4c6v7mT2gdlk5GSU+LhCiOKztvRKM6XUXuAQcFgptVspJddQKjmtNQkzZuAaEYFf\n3z7F3v/Lo1+y/tR6nmn1DM1CCqlIvG8p551d+MvNvdSxhvuE81bHt/iy35e0qd6Gj/d+TJ+VfVjx\n1wpyTDmlPr4Q4sasPeU1C3hRa11bax0JvATMKWmnSqlApdR6pdRRy3OhBZuUUkal1D7LY1WB7QuU\nUicKfNaipLGIoqX/spWsgwcJevwxlGvx5rH+nfw3U36dQvvw9jzc5OFrGySdhJNb2OTlCyWc01KY\nmwJu4uNuH7PozkVE+kby1o63GPjNQNaeXItJm2zWjxDiWtYmFG+tdf41E631JsC7FP2OAzZqreth\nvg25qNWTMrXWLSyPqws7jS7wmdw/amN5oxOX8HD8Bwwo1r4FS6tM6jQJJ1XIX7P9ywDFZk+faz+z\ngZahLVnQawGfdPsEVydXRv88mvvX3M+O8zvs0p8QwvrikMeVUq8Biy3vHwJOlKLfu4DbLK8XApuA\nsaU4nrCxjJ2/krlnD2GvTUC53fjuq4Le++09jiYd5dPunxLiFXJtA5MJ9i2F6C7sMdpvjTelFF1q\ndaFTzU6sObGGT/Z+wmPrHqNDeAeeb/08jYMaX3f/pHQDRy6mciQujaMXU7kUeycmoyczNv3Nw7dE\n4ekmNzoKUZC1/5r/D3gD+ApQmCc6PlKKfsO01ucBtNbnlVKhRbTzUErtAnKByVrrrwt89o5S6nUs\nIxyttRR7sqGEGTNwCQnB/557irXfj7E/suyvZQxrPIxbI24tvFHsNkg+Bd0mwO7lNoj2+pydnOlf\ntz89o3qy/M/lzDk4h3tX30uvqF480/IZ/FzDOXoxlSMX08wJxPI6Ie2fv1I+7i6gnHFyzmTK2j/5\nbOsJnu12E/feHImbiywrJARYmVC01klAse7qUkptAKoX8tGrxThMpNb6nOU25R+VUge11n9jvoX5\nAuAGzMY8uil0JSal1OPA4wCRkZHF6Lrqytizh4ydOwkdNxYnd+svmF9Iv8Dr216nUWAjnmtV9CRF\n9i0FN19o2LdMEkoeQ44TTXz68Wjtm/kudhnrTqxm7Yl1GJLbYojvjjb64uXmTL0wX7o2CKF+mC/1\nwnyoH+ZLuJ8H7ReYk+vHt81h6g9/8to3vzNnywleuL0e/WNq4uxku2tBQlRE100oSqlvgSKLJhVy\nXaPgZ0VOqVZKXVRKhVtGJ+FAXBHHOGd5Pq6U2gS0BP7OG90A2Uqp+cDL14ljNuakQ5s2baQAlBUS\nPp2Bc2AgAUOGWL2P0WTklS2vXFNa5RrZafD719BsELh52SjiK6Vn53I0Lo0jF1KvOGV1PiUrv42H\na3vqhLUF//WcC9iET9A+Bta9n2dajcDPo9p1j9+2TiArnujApiPxTF37Fy8s38/MTcd5uWcDejQK\nLXHhTCEquhuNUN6zU7+rgOHAZMvzN1c3sNz5laG1zlZKBQMdgXctn+UlIwUMwHw7s7CBzAMHSP/l\nF0JefBEnL+t/4M85OIddF3fxTqd3riitco0/VkFOOrR4sNSxZhhyORaXxpGLaZZTVuZTVWeTM/Pb\nuLs4cVOoD+2jg8yjjVBf6of5EhHgiZOTAvoQezmWj/d+zPKj8/khdiWPNXuMexvei7tz0aMzpRRd\nG4TSpV4Iaw6e5/31R3hs0S5aRvozumcDbqkbXOrvJ0RFc92EorX+2U79TgZWKKUeBWKBwQBKqTbA\nSK31CKARMEspZcJ8N9pkrXWCMFaqAAAgAElEQVTewuOfK6VCMF/P2QeMtFOcVU7CjJk4+fkR8MAD\nVu+TV1qlT3Qf+kX3u37jfUshMBpqtbP6+Fk5RkviSP0necSlciYpk7yiw27OTkSHeNO6dgD3t61F\nvTBfGoT5UivQ64anoiKrRTK1y1QebvowH+7+kKm7prLkjyWMajGKvtF9cXYq+uK7k5OiX0wNejWt\nzhe7z/DhhqM8MGcnt9YLZnTPBjSP8Lf6ewpR0Vl1DUUpdZBrT32lALuAt7XWicXp1NK+eyHbdwEj\nLK+3AYXMhgOtdbfi9Cesk/XHH6T99BPBzzyNs491d4WnZKcwdvNYanjXYEK7Cdc/3WOZe0K3CYXO\nPcnKMXI8Pp2jcalXJI/YSxmYLH/7XJ0V0cE+xET4M7h1LeqH+VAvzJfagV64OJfu4nheufwd53cw\nffd0JmydwILfF/B8q+fNa7dch6uzE/e3jWRgy5os2XGK//x0jP6fbKVXk+q83LM+N4XKAqei8rP2\nLq/vASOw1PL+PsyjgxTMZexv8GupqAgSZszEyceHwKFDrWqvteaN7W8QnxHPojsXXVta5WqWuSc0\nvy9/U3Z6DTKTG9LtvU2cTEzPTxwuToqoYG8a16jGXS1qUj/Ml/phPkQFe+NaysRxI3nl8tedWsdH\nez7i6R+fplVoK4xk4ozndff1cHVmxK3R3HtzLeb9coK5W06w7vAF7m4VwfM96hERYJ/rRkKUB9Ym\nlI5a64Jrvh5USm3VWndUSj1kj8BE2co+epTUdesIGvkEztWuf1E6zxdHv2D9qfW82PrFwkurFFRg\n7gn+tQDziCTl/G0oTNSv7Uvf5uHUCzNf46gT7O3Q23HzyuV3i+zGyqMrmbF/BtlOCThrb7ac2UKH\nGh1wcSr6n4+vhyvP96jPsA5RfPrTMRbtOMU3+87yYLvajOp6EyG+pS83I0R5Y21C8VFKtdNa7wRQ\nSrUF8n4dzbVLZKJMJcychfLyInD4cKvaH0s6xpRfp9AhvAPDm1ixT8G5Jxbf7j+HNnrgX+s7Zg4d\nXNLQ7SqvXH7f6L50WtKbHJXEUxufItAjkF5RvegT3Ydmwc2KPNUX6O3GhL6NefTWOny08SiLd5xi\nxa7T/F/HOjzWORo/z+KVtBGiPLM2oYwAPlNK+WA+1XUZeFQp5Q38217BibKRfeIEl7//nsBHHsYl\noNCyalfIys1izJYxeLt6M+nWIkqrXK3g3BPMp8sWbj+Js1sSrp4XSvkN7M/L1QtXgnDRAUzu9gyr\nj6/miyNfsPTPpUT6RtI7ujd96vQp8g63cD9P/n13cx67NZr31x/hk5+OsXjHKZ68rS7DO8ise1E5\nWDux8TegmVLKD/PqjQUXnFhhl8hEmUmcNRvl5kbQI9YVP5i2axpHk44yo8cMgj2tuD22kLkne2KT\nOXT2Mr5hh21ZG9LuFE50i+xGt8hupBpS2XBqA2tOrGHW/lnM3D+TpkFN6RPdh151ehX63yY6xIdP\nHmjFyC4pTFv3F5O//5PPfjnBM93rcW+bWjLrXlRo1pav91NKvY+5zMkGpdQ0S3IRFZzh9GlSvv0W\n/yGDcQm+cXLYGLuRZX8tY3jj4XSq2cm6TgqZe7Jo+0l83V3wqPZ3CSN3PF83XwbWG8jcO+ay/p71\nvNzmZYzayJTfptD9f915Yv0TfPv3t6TnpF+zb9Oafsx/pC0rnuhA7SAvXvv6ED3e/5mVe89gNMn8\nW1ExWfvr0GdAKjDE8rgMzLdXUKLsJM6eg3JyIujRR2/Y9kL6BV7f+jqNgxpfv7TK1a6aexKXmsV3\nB89zT5sInJwqxyW4MO8whjcZzop+K/j6rq95tOmjnLp8ivG/jOe25bcx5ucx/Hz652vWZsmbdT//\n4ZvxdnfhheX76f3hFtYfvojWklhExWLtNZS6WutBBd6/oZSSkvEVXM65cyR//TX+9wzCNSzsum2N\nJiPjtowjx5TDu53fxdXZyovJhcw9+e/O0+QYNcM6RPHdt6X8EuVQXf+6PNvqWZ5p+Qz74vex5vga\n1p5cy/cnv8ff3Z+eUT3pG92XmJAYlFLmWfcNQ+lS/8pZ960i/RndsyEd6gY5+isJYRVrE0qmUqqT\n1voXAKVURyDzBvuIci5x7jzQmuARI27Yds7BOey+uJt3Or1D7Wq1re/kqrknOUYTn+88RZf6IdQJ\nLs2SOuWfUoqWoS1pGdqSsTePZeu5raw5voavj33N8r+WU9OnJr3r9KZvdF+i/aMLnXV//5wdMute\nVBjWJpQngYV5F+WBS8DD9gpK2F9OXBzJX3yB34C7cK1Z87pt91zcw4z9M+gb3Zf+dYusB3qtQuae\nrD10gbjUbCYPKkZSqgRcnV25rdZt3FbrNtIMaWyM3cia42uYd2gecw7OoVFgI/pE9+HOOncS6hVa\n6Kz7O5tW56U7ZNa9KL+svctrHxCjlKpmeX/ZrlEJu7s07zO00Ujw449ft11Kdgpjt4ylpk9NXm1X\nnJUHKHTuyaLtJ4kM9OK2+kUtgVP5+bj5cNdNd3HXTXcRnxHP2pNrWXN8De/teo9pu6bRNrwtfaP7\n0iOyR/6s+7lbTjB3y3F++P0Cg1pF8JzMuhfl0I3K179YxHYAtNbv2yEmYWe5iYkkLV+OX98+uF1n\njZi80ioJGQks7r34xqVVrnbV3JPfz6Xw28kkJvRpZKn0C1GGIlceqBJCvEIY2ngoQxsP5UTKCdYc\nX8Oa42t4betrvL3jbbpEdKFPdB+e7nYrw28pOOv+HA+0i+TpbjcR7COz7kX5cKMRioytK6FLCxag\ns7MJeuKJ67YrWFqlaXDT4nVSyNyTRdtO4enqzODWtUoaeqVWx68OT7d8mlEtRrE/fj9rjq/hh5M/\nsO7UOvzc/bij9h30ubkPD3fszCc//n3FrPvHu0RTzUNm3QvHulH5+jfKKhBRNnKTkkj6fCnV7uyF\ne3R0ke3ySqvcUuMW60qrXO2quSfJGQa+3neWu1tF4OclP/iuRylFi9AWtAhtwZi2Y9h+bjtrjq9h\n9fHV/O/I/wj3Dqd3nd7Ma9mNL3bkyqx7UW5Ye1E+n1Jqj9a6lT2CEfaXtHgxpowMgp4oegmZrNws\nRm8ejberN+90ese60ipXu2ruyYpdp8nONTGsQ9W6GF9ark6udI7oTOeIzmTkZJgv5p9Yw4LfF2DU\n82gQ0IAn+nXn4JE6+bPun+1ej3tvllGgKHvFTiiY7/ISFZAxNZVLi5fge3sPPBrUL7Lde7ve41jy\nMetLq1ztqrknRpNm0fZTtK0TSKNw6yoZi2t5uXrRr24/+tXtR0JmAj+c/IE1x9ew9NinKCdFm/Yx\nJMc1Y8KqZGZvPk6mezQevscdHbaoQkqSUNbYPApRJpKWLMGUmkrQyKJHJxtjN7L8r+XFK61ytavm\nnvz0ZxxnkjIZ37tRyY4nrhHsGcyDjR7kwUYPcuryKb47/h1rTqzhovtiAhq6kpXdhIy4pqTGtaL1\n29/j5+lBgJc7/p6u+Hm64udlfvb3dMXfyy1/W/7nnq6lXrBMVD3FTiha6wk3biXKG2NaOpcWLMSn\nSxc8mzQptE2JS6sUVMjck4XbT1K9mge3N752Nv7yJzqUrB+Rr3a12jzZ4klGxozkUMIh1pxYw/cn\nvsfTzVzMwgAk4Mwl7Q7aHZ3qjinZjdxcN7TJHUzuaJM72uRW4LU7Hs6eeLt44+vuja+7DwEePgR4\n+BLoVY0gb6/8hJSfnCyJydvN+ford4pKy9olgFMpegngl7TWMq4u55KX/RdjSgrBTz1Z6OdGk5Gx\nm8eSa8otXmmVq1019+Tv+DS2HE3gpdvr232lxapOKUWzkGY0C2nGy21epv3Cvpgw8FSbB0jPSc9/\nZORkkJGbQZohnVRDGuk5iaTnZJBlzMCor6ytlmF5XATzmq3p5oeOcwajO1q7oY1XJiWlPXBz8sTd\n2RMvFy+8Xb3xcfWmmrsPfu4+BHj6EOTlS5B3NUK9/Qjz9cOU64FyMmA0aZwUkpAqKGtHKO8D5zAv\nAawwLwFcHfgLc+HI2+wRnLANU2YmifMX4H3LLXjGxBTaZvbB2eyJ28OkTpOKV1rlalfNPVm8/RRu\nzk7c367o+S7C9lycXHDGG2e8GdHsxqV18hiMBjJyMkjP/Sf5XJGMcs3vU7LSSMpKIyUrlVRDOmkG\n82eZualkGy9iMGWSRRaZGEkE8zJ8uZgTUiG0izNoN5rPzxuxKvIu1yoU6H/em7eoAm0KtLM8VCFt\n8z9XeZ86FdjT0kaZ+1Iqb4vllyClUJb2CkVqtgE0dFpQ9IK1/8RQ+Kf5rwptdv2EWvSn6ppXaVnm\ngqSHLsTStLp9/x1am1B6aa3bFXg/Wym1Q2v9plJqvD0CE7aTvGIFxsREgp8s/NrJnot7mLl/Jn2j\n+9Kvbr+Sd3TV3JO07Fy+2H2GPs3DZfJdBeHm7Iabsxv+lL5umNYag8lwRVJKzU4jISOV+IzLXMpI\nJSkzlZTsNDac2IoGmgTfhMaE1qDRmLTJUnVZo7XGhDb/T5uf0RoTpvzKzNryWgNa57X+ZztXvNb5\n/zP/30TeVvL3zf2njaVf0CinLAAyTcaivn0xthanlabIItTqimb/cDYAkG7Itqr30rA2oZiUUkOA\nLyzv7ynwmdTYLsdM2dkkzp2HV5s2eN188zWf55VWifCJYEL7Ul4eu2ruyVd7zpCWnSu3CldRSinc\nnd1xd3YnwOP6K4G2m78dgBX3TC2L0Eqt3Xxz8fWdj3zp4EhuLC/WdpH17N6XtSe1HwSGAnGYT6cO\nBR5SSnkCT9spNmEDyV9+SW58fKHXTrTWTNw2kYSMBN7t/C7erqWs/ltg7onWmoXbThIT4UfLyBsv\nKyyEqPisLQ55HCjqXMgvtgtH2JI2GEicMxfPmBi8Olx7N9X/jvyPDbEbeKn1SzQJLvzOL6tdNfdk\n27EE/o5PZ9rgwq/ZCCEqH2uXAK6vlNqolDpked9cKSW3D5dzKatWkXv+PMFPPXnNXTPHko7x7m/v\n0rFGR4Y1GVb6zq6ae7Jg20kCvd3o0zy89McWQlQI1p7ymgO8AuQAaK0PYL7TS5RTOjeXhFmz8WjS\nBO/Ona/4rGBplbc7vV2y0ioFXTX35PSlDDb+cZH729bCw1XqSglRVVh7Ud5La/3rVb/lVo7FwCuh\nU0OHkZuQQM7p04R98vE1o5O80ioze8wsWWmVq1019+TznbEAPNhOLsYLUZVY+6tpglKqLpY7upRS\n9wDn7RaVKBWtNTnnz+Nevz4+3bpd8dnGU+bSKg83eZiONTvapsMCc0+ycows+y2WOxpXp4a/p22O\nL4SoEKwdoYwCZgMNlVJngROY7/wS5ZDx0iV0VhbBT45EOf3zO8OF9Au8vu11mgQ14dmWz9qms6vm\nnqzadZrkjByG3xJlm+MLISoMa0coZ4H5wDvAMmA9UIJFMsyUUoFKqfVKqaOW50LvK1VKRSql1iml\n/lBKHVZKRVm211FK7bTsv1wp5VbSWCobbTKRduYkOa5O+N5xR/72XFOubUqrXK3A3JO8W4Xrh/nQ\nPjrQNscXQlQY1iaUbzDfNpyDuQRLGkUWUbDKOGCj1roesNHyvjCLgKla60ZAW8zzYACmAB9Y9k8C\nHi1FLJVK8or/4WYwkRLghnL+54L4nANz2BO3hwntJxBZzYblFwrMPdkTm8Tv5y4zrEOU1GISogqy\n9pRXhNa6lw37vYt/6n8tBDYBYws2UEo1Bly01usBtNZplu0K6AY8UGD/icAMG8ZXIWUfP8HFyZPJ\n9HQmw+efP9rdF3cz88BM+kX3K11platdNfdk4bZT+Hq4MLBlTdv1IYSoMKwdoWxTSjWzYb9hWuvz\nAJbn0ELa1AeSlVJfKaX2KqWmKqWcgSAgWev8sqhngCr/E0zn5HBuzBic3N25FOqRX3EuJTuFcVvG\nEeETwavtX7VtpwXmnsRdzuK7g+cZ3LoW3u4lWWZHCFHRWZtQOgG7lVJ/KaUOKKUOKqUOXG8HpdQG\npdShQh53WdmnC3Ar8DJwMxANPEzhhTaLrCemlHpcKbVLKbUrPj7eyq4rnvj//IesQ4eo/uabGF3M\nf6z5pVUybVRapaCr5p4s/TWWXJNmqNTtEqLKsvZXyTuLe2CtdY+iPlNKXVRKhWutzyulwvnn2khB\nZ4C9eWutKKW+BtpjLpfvr5RysYxSIjBf1ykqjtmY71CjTZs2lbKQZcbu3STOnoPf3XdTrecd8JF5\nPkheaZWX27xc+tIqVysw98SQa+LznbHc1iCEOsE2TFpCiArF2lpep2zc7yrMd4lNtjx/U0ib34AA\npVSI1joe83WTXVprrZT6CXPF42XX2b9KMKamcm70GFxr1iRsvHklgWXPNCEzN5NjltIqQxsPtX3H\nBeae/PD7BeJTsxneIcr2/QghKgxHLaE3GbhdKXUUuN3yHqVUG6XUXACttRHz6a6NSqmDmE91zbHs\nPxZ4USl1DPM1lXllHH+5cfHtt8m5eJEa707B2cc8OjBpE38n/2270ipXy5t70nQguHmxcNtJagd5\n0aV+iG37EUJUKA65eqq1TgS6F7J9FzCiwPv1QPNC2h3HfBtxlXb5u+9I+WYVwaNG4dWyZf7206mn\nyTJm8WHXD21TWuVqBeaeHDqbwq5TSUzo0wgnJ7lVWIiqTBb5rqByzp/n/MQ38IhpfsVKjGtPrCU+\nM54wrzBuqXmLfTovMPdk8fZTeLo6M7h1Lfv0JYSoMCShVEDaZOLcuFfQubnUfPddlIt5oPnXpb94\nfdvr+Lj6UNPHTndS5809afEAyZk5fL3vLANa1sTPy0Yz74UQFZZMGKiALs1fQMbOnYS//RZutc23\n6aZkp/D8T8/j6+pLuHe47a+b5Ckw92T5b6fJzjUx/Ba5VVgIISOUCifrjz+Imz4d39t74DfIvFa0\n0WRk7OaxXMi4wPtd37ddna6rFZh7YqwWweIdp2hXJ5CG1avZp79ypnF4NRqHV43vKkRJSEKpQExZ\nWZwdPRoXf3+qv/lmfr2sT/Z9wtZzW3m13avEhNhxyd28uSctHuSnP+M4k5QpVYWFEPkkoVQgce9N\nw3Dsb8L//W9cAswFmtedXMfcg3O5p/493FP/HvsGUGDuycLtJwn38+COxmH27VMIUWFIQqkg0rZs\nIWnJEgKGDcWnk3lhrGNJx5iwdQLNQ5rzSttX7BtAgbknx5JNbDmawIPtInFxlr9CQggz+WlQAeRe\nusS58eNxr3cToS++CMBlw2We++k5vF29+eC2D3BztvOSMAXmnizZcQo3Zyfua2vDMvhCiApP7vIq\n57TWnH/tdUzJKdSYMwcnDw9M2sQrW17hXNo55vWcR6hXYcWabcwy9yQttDVf7P6Rvs3DCfZxt3+/\nQogKQ0Yo5VzyF1+QtnEjIS+8gEfDhgDM2D+DzWc2M7btWFqFtbJ/EAXmnny19yxp2bkMk4vxQoir\nSEIpxwwnT3Jx0r/xat+ewIfNKy7/GPsjM/fPZMBNA7i3wb1lE4hl7olufi8Lt50kJsKPFrX8y6Zv\nIUSFIae8yimdk8PZMWNRbm7UmPxvlJMTx1OOM/6X8TQJasKE9hOKXGZ3fq/5tgukwNyTrfFe/B2f\nzvtD7HhrshCiwpIRSjmVMGMGWQcOEP7GRFyrVyfNkMZzPz6Hu7M707tOx925jK5fFJh7snD7SYK8\n3ejdLLxs+hZCVCiSUMqhjD17SZg5C78BA6jWqxcmbWL8L+M5nXqa97q8R3Xv6mUXjGXuyZmwbmz8\n4yL3ta2Fh6tz2fUvhKgw5JRXOWNMS+PcmDG41qhB2ATzGvCzD8zmp9M/Ma7tOG6ufnPZBZM396TZ\nIBbviUcpxYPtqm7dLpueShSiEpKEUs5cfPsdcs6do/aSxTj7+LD5zGY+3fcpfaP78kDDB8o2GMvc\nk+ym97F8yWnuaBxGDX/Pso1BCFFhyCmvcuTy2rWkfP01QU88jlerVpy6fIpxm8fRMLAhr3d4vciL\n8HZjmXvyTWItkjNyGCZL/AohrkMSSjmRc+EC5/81EY9mzQh56inSc9J57sfncHZy5oOuH+DpUsYj\nA8vcEx3zAAu3n6JBmC/towPLNgYhRIUiCaUc0CYT5155BW0wUOPdKeDiwmtbX+PE5RNM7TLVfotl\nXY9l7snB4F78fu4yw26pXfYjJCFEhSIJpRy4tHARGdt3EPbKONzr1GHeoXmsP7WeF1u/SPvw9mUf\nUIG5J3MP5OLr4cKAFg5IakKICkUSioNl/fUX8e+/j0/37vgPHswvZ3/hoz0fcWfUnQxrPMwxQVnm\nnqQ0GMx3B88zpE0tvN3l/g0hxPVJQnEgU3Y2514ejZO/H+FvvcmZ1DOM2TyGegH1mHjLRMedYrLM\nPVmc3Jxck2Zo+6p7q7AQwnrya6cDxU2bRvbRo9SaMxuDrwfPfT8ChWJ61+l4uXo5JijL3BNjk0Es\n3BXHbQ1CiAr2dkwsQogKRRKKg6T9spWkRYsJePBBvDt1YszmMRxLOsaMHjOo5VvLcYFZ5p5sr9aT\n+NRsWeJXCGE1SSgOkJuUxPlXXsGtbl1CR7/MosOLWHtyLc+1eo6ONTs6NjjL3JPpfwZQO8hAl3oh\njo1HCFFhyDWUMqa15sLr/yI3OZma701l56W9vL/7fW6vfTuPNn3UscFZ5p5cjB7ErthkhravjZOT\n3CoshLCOJJQylvLVV6SuX0/o88+RWKsaYzaPIdovmrc7vu34eR6WuSfzLrfF09WZwW0ceOpNCFHh\nyCmvMmSIjeXCO5PwatcOz6H3MfKHhzGajI69CJ/HMvckp3ZnFh42Mqh1BH6ero6NSQhRocgIpYzo\n3FzOjR6DcnEh/N+TeHPnW/x16S8md55M7Wrl4LZcy9yTzV63k51rYliHchCTEKJCcUhCUUoFKqXW\nK6WOWp4DimgXqZRap5T6Qyl1WCkVZdm+QCl1Qim1z/JoUZbxl0TCzFlk7t9P+MR/sSL5R9YcX8Oo\nFqPoHNHZ0aGZ7VuKdvNl0ombaB8dSMPq1RwdkRCignHUCGUcsFFrXQ/YaHlfmEXAVK11I6AtEFfg\ns9Fa6xaWxz77hls6mfv2kTBjBtX69+OvViG8t+s9utbqymPNH3N0aGaWuSdna/bi72QTw6WqsBCi\nBByVUO4CFlpeLwQGXN1AKdUYcNFarwfQWqdprTPKLkTbMKalc3b0GFzDwnB68XFe/vllavnWYlKn\nSTipcnLG0TL35LP0Wwj38+D2xmGOjkgIUQE56idamNb6PIDlObSQNvWBZKXUV0qpvUqpqUqpgmvP\nvqOUOqCU+kApVUYLrBffxX9PIufsWYInv80LuyaQbczmw24f4uPm4+jQ/rFvKQa/KD6LDeWh9rVx\ncS4niU4IUaHY7SeHUmqDUupQIY+7rDyEC3Ar8DJwMxANPGz57BWgoWV7IDD2OnE8rpTapZTaFR8f\nX9KvUyKX160j5cuvCHpsBO8Z1vB74u9M6jSJaL/oMo3juixzT7Z43YGbszP33iy3CgshSsZutw1r\nrXsU9ZlS6qJSKlxrfV4pFc6V10bynAH2aq2PW/b5GmgPzMsb3QDZSqn5mJNOUXHMBmYDtGnTRpfs\n2xRfzsU4Lrz2Oh5Nm/Lj7aF8s/szRsaMpFtkt7IKwTr7l6FR/PtsDH2bhxPsU24He0KIcs5R5zZW\nAcMtr4cD3xTS5jcgQCmVV/ujG3AYwJKEUOaZgAOAQ3aNtpi0ycT5V17BZDCQPHY4U/a8R5eILjwZ\n86SjQ7uSZe7J+cB2HDMESN2uSqZxeDUah8vdeqLsOGpi42RghVLqUSAWGAyglGoDjNRaj9BaG5VS\nLwMbLYljNzDHsv/nlkSjgH3AyDL/BteRtGQJ6du24TX+RUYef4+avjWZdGs5ugifxzL3ZL7H3cTU\n8iemlr+jIxI2NL/XfEeHIKoYhyQUrXUi0L2Q7buAEQXerweaF9KunJ03+kfWX0eIe28aXrd1YXzg\nT2SkZDD3jrlUcyuHvynuW0quiw+Lk5sz6Q6ZyCiEKB0pvWJDpuxszo0ejZOvL0sH+HPg4lbev+19\nbgq4ydGhXcsy92SbRxe8tS99moc7OiIhRAUnCcWG4j+YTvaRI5x8/SGWXlzGiGYjuL327Y4Oq3CW\nuScfpd/M/V0icXdxvvE+QtjJzke+dHQIxVKR4i3LWMvZSf2KK33bNi4tWIBxwO2MN35BxxodebrF\n044Oq2j7lnLJPYK9NODB9pGOjkYIUQlIQrEBY3Iy58a9gnNUJC81PUh1r+pM6TwFZ6dy+lu/Ze7J\nUkMnejapTrifp6MjEkJUApJQSklrzfl/TST30iXm3O1Dkkpnetfp+Ln7OTq0olnmnizNvIVhUrdL\nCGEjcg2llFK+/obUH37g4OAWrHU/xNRbptIgsIGjwyqayYTet5R9LjH4htWhXZ1AR0ckhKgkZIRS\nCobTp7n41ltkNIni7eiDPNzkYXrV6eXosK4vdhsq+RQLMm5h+C1Rjl8lUghRaUhCKSGdm8u5MWMx\nOsG4rudpW7M9z7V6ztFh3di+pWQ6ebHNrQMDWtZwdDRCiEpEEkoJJcyeTebevSzs5Y5T9TCmdp6K\ni1M5P4OYnYbp95WsymlH/zY34eVWzuMVQlQoklBKIHP/fhL+8ymHWwexsaGB6V2nE+BR6KKT5csf\nq3DKyeB/xs4MbS8z44UQtiW/ohaTKT2ds2PGkOHvwbudk/lXh3/TKKiRo8Oyimnv55yhOr43dSQq\n2NvR4QghKhkZoRTTxcmTMcSeZkqvLO5uOZR+dfs5OiTrJJ3E6dQvLM/pzLCOdRwdjRCiEpKEUgyp\nGzaQ/L8vWN3BBZ+2bXmxzYuODsl6+5dhQvFbtdvpUi/kxu2FEKKY5JSXlXLi4jg7YQKna7jy4x2h\nfN55Kq5Oro4OyzomE4bdS9hpbELPjjfj5CS3CgshbE9GKFbQWnNu/HgM6al81M+ZaT0+JMgzyNFh\nWS92G26pp1mlbuOe1mgOxaoAAAjLSURBVBGOjkYIUUnJCMUKf/fsRU5sLAvvcGJEv3/RJLiJo0Mq\nluxdi8nRnnjHDMDPs4KMqoQQFY4klBvQWnM86yxxdRWB99/PgJsGODqk4slOw+mPb/jW2J77OzZ0\ndDRCiEpMEooV3hzph3OOkQ1txzo6lGIzHf4GV2Mmf4T15f7qvo4ORwhRiUlCuQGlFDcF1MOojbg6\nV7zTRcnbFpBiCuOW23o7OhQhRCUnCcUKC+5c4OgQSibpJIHxv7LC9QFGNK7u6GiEEJWc3OVViSVu\nW4hJKzzbPICLs/xRCyHsS37KVFYmE2r/MrbrpvS9ta2joxFCVAGSUCqpjGObCTSc40StuwjycXd0\nOEKIKkASSiV1btM8UrUnzXs85OhQhBBVhCSUSsiUlUrNc+vY7tmZ5nXCHR2OEKKKkIRSCR39eSme\nZOHeRkYnQoiyIwmlEtJ7PyeW6rTvInNPhBBlR+ahWGHnx8MJS9rt6DCs1tB0ml8iRxLpKn+8Qoiy\nIz9xrGCqVpNL2ZccHYbV4p0b0bjvs44OQwhRxTgkoSilAoHlQBRwEhiitU66qk1X4IMCmxoC92mt\nv1ZK1QGWAYHAHmCo1tpgr3g7DJ9kr0MLIUSl4ahrKOOAjVrresBGy/sraK1/0lq30Fq3ALoBGcA6\ny8dTgA8s+ycBj5ZN2EIIIYriqIRy1/+3d/+hftV1HMefr5bLaIbgpi23tX6MyKkruKkxifJHrJJF\nai2xWJT4T4JRYIxFIhIIg+iPilxLElqLUqeyKTrz6iWotlVz3nFdzVE5mm4FVkMrda/+OJ+bt3Xv\n93u/2/fcc776esBl53x3Pt/zupe773vn1+cN3F6Wbwe6zQl/JXC/7eckiarA3NHD+IiIqFlTBeUM\n2wcByp+nd9n+U8Cmsnwa8KztF8v6AeDMWlJGRMS01XYNRdJDwGRT3K7t8X3mA+cAD4y/NMlm7jD+\nWuBagEWLFvWy64iI6EFtBcX2JVP9naRnJM23fbAUjEMd3uqTwGbbL5T1vwCnSnptOUpZAPy5Q471\nwHqAoaGhKQtPREScmKZOed0LrC7Lq4F7Omx7FS+f7sK2gWGq6yrTGR8RETOgqYJyC3CppN8Dl5Z1\nJA1J2jC+kaTFwELg0WPGfwX4kqR9VNdUvj8DmSMiooNGnkOx/Vfg4kle3wlcM2H9D0xywd32fiBN\nPiIiWkTVGaRXB0mHgT8e5/C5VNdvBsUg5U3W+gxS3kHKCoOV90SzvsX2vG4bvaoKyomQtNP2UNM5\npmuQ8iZrfQYp7yBlhcHKO1NZM9twRET0RQpKRET0RQrK9K1vOkCPBilvstZnkPIOUlYYrLwzkjXX\nUCIioi9yhBIREX2RgtIDSTdL2i1pl6QHJb256UxTkbRO0hMl72ZJpzadqRNJn5C0R9JRSa28c0bS\nCkl7Je2T9H8tF9pE0m2SDkkabTpLN5IWShqWNFZ+B65vOlMnkk6WtF3SYyXvTU1n6kbSLEm/lbSl\nzv2koPRmne1zS4+WLcDXmg7UwTbgbNvnAr8D1jScp5tR4HJgpOkgk5E0C/g28GHgLOAqSWc1m6qj\nHwArmg4xTS8CX7b9LuAC4Ast/9n+C7jI9jLg3cAKSRc0nKmb64GxuneSgtID23+fsPoGOsxy3DTb\nD06Y4v+XVJNotpbtMdt7m87RwXnAPtv7S3fQH1P19Wkl2yPAQPSttn3Q9m/K8j+oPvha25LClSNl\n9aTy1drPAkkLgI8CG7pte6JSUHok6euSngKupt1HKBN9Dri/6RAD7kzgqQnr6cNTgzJ/33uAXzWb\npLNyCmkX1Uzp22y3Oe83gRuAo3XvKAXlGJIekjQ6ydfHAGyvtb0Q2Ahc1+asZZu1VKcUNjaX9L9Z\nuuZtsZ768ETvJM0B7gS+eMzZgNax/VI59b0AOE/S2U1nmoyky4BDtn89E/trZHLINuvUx+UYPwK2\nAjfWGKejblklrQYuAy52C+4P7+Fn20YHqGa+HtexD0/0RtJJVMVko+27ms4zXbaflfQI1fWqNt4A\nsRxYKekjwMnAGyX90Pan69hZjlB6IGnJhNWVwBNNZelG0gqqaf5X2n6u6TyvADuAJZLeKmk2VVvq\nexvO9IogSVQtKMZsf6PpPN1Imjd+16Sk1wOX0NLPAttrbC+wvZjqd/bhuooJpKD06pZyimY38CGq\nOyfa6lvAKcC2cpvzd5sO1Imkj0s6ALwP2CrpgW5jZlK5weE6qlbUY8BPbO9pNtXUJG0CfgG8U9IB\nSZ9vOlMHy4HPABeV39Vd5X/UbTUfGC6fAzuorqHUejvuoMiT8hER0Rc5QomIiL5IQYmIiL5IQYmI\niL5IQYmIiL5IQYmIiL5IQYnoI0lHum/Vcfwdkt5WludIulXSk2VW2xFJ50uaXZbzYHK0SgpKREtI\nWgrMsr2/vLSBaoLHJbaXAp8F5pbJKX8GrGokaMQUUlAiaqDKuvIg7OOSVpXXXyPpO+WIY4uk+yRd\nWYZdDdxTtns7cD7wVdtHAcpMx1vLtneX7SNaI4fMEfW4nKpXxjJgLrBD0gjVU+GLgXOA06meur+t\njFkObCrLS4Fdtl+a4v1HgffWkjziOOUIJaIeFwKbyqy0zwCPUhWAC4Gf2j5q+2lgeMKY+cDh6bx5\nKTT/lnRKn3NHHLcUlIh6TDbdfafXAZ6nmhEWYA+wTFKnf6OvA/55HNkiapGCElGPEWBVacQ0D3g/\nsB34OXBFuZZyBvCBCWPGgHcA2H4S2AncVGbjRdKS8d4xkk4DDtt+Yaa+oYhuUlAi6rEZ2A08BjwM\n3FBOcd1J1VtlFLiVqjPh38qYrfxvgbkGeBOwT9LjwPd4uQfLB4H76v0WInqT2YYjZpikObaPlKOM\n7cBy20+X3hrDZX2qi/Hj73EXsMb23hmIHDEtucsrYuZtKQ2aZgM3lyMXbD8v6UaqXvV/mmpwafB1\nd4pJtE2OUCIioi9yDSUiIvoiBSUiIvoiBSUiIvoiBSUiIvoiBSUiIvoiBSUiIvriPwuxUG9TygkA\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x8940be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#pd.DataFrame(grid.cv_results_).to_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#cvresult = pd.DataFrame.from_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#test_means = cv_results['mean_test_score']\n",
    "#test_stds = cv_results['std_test_score'] \n",
    "#train_means = cvresult['mean_train_score']\n",
    "#train_stds = cvresult['std_train_score'] \n",
    "\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'neg-logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的正确率（score）。可以看出在训练集上C越大（正则越少）的模型性能越好；但在测试集上当C=100时性能最好（L1正则和L2正则均是）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(train_X, train_y) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.60306301, -0.65080486, -0.65724208, -0.657913  ],\n",
       "        [-0.60073965, -0.64616371, -0.65143281, -0.65195748],\n",
       "        [-0.48023569, -0.49535331, -0.49747474, -0.49769213],\n",
       "        [-0.58967698, -0.61249814, -0.61567097, -0.61600958],\n",
       "        [-0.46299191, -0.48160967, -0.48418537, -0.48444479]])}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.60306301 -0.65080486 -0.65724208 -0.657913  ]\n",
      "[-0.60073965 -0.64616371 -0.65143281 -0.65195748]\n",
      "[-0.48023569 -0.49535331 -0.49747474 -0.49769213]\n",
      "[-0.58967698 -0.61249814 -0.61567097 -0.61600958]\n",
      "[-0.46299191 -0.48160967 -0.48418537 -0.48444479]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl8VdW5//HPk4QwzwQZw6AogyJC\nRK0zrV6cwKrXoZNYrdfeS+2grdrW4tBavba1/tTbOrZ0Ulo6BWdxrG1VwiAaJiMIhDEMYSbj8/tj\n78AxJuRAsrPPSb7v1+u82MPa+zyLA/vZa+1hmbsjIiJyqDLiDkBERNKbEomIiDSKEomIiDSKEomI\niDSKEomIiDSKEomIiDSKEomIiDSKEomIiDSKEomIiDRKVtwBNIdevXr54MGD4w5DRCStzJ07d5O7\n5zRUrlUkksGDB1NQUBB3GCIiacXMViZTTl1bIiLSKEokIiLSKEokIiLSKEokIiLSKEokIiLSKEok\nIiLSKEokIiLSKK3iORIRkVRWXe2UV1VTUVVNRZWHf+6fLq/cP19ZVR2W3V8uWO9UVu+frll31clD\n6NExO9L4lUhEpMWprnYqqquprDkQ1xx4Kz95sP7EuurEctWU15StDNfVTCeuqwq+a38yqKaiMpiv\nrA6m939XwvdVB9NV1R7J34MZTB7TT4lERFLf+m172bhjb3h2HJwZ10xXVH38oFyZcDAtTzhg1z4T\nL084YFdW+8fOyhMPypUfO2v3feWjkp2ZQZtMIyszgzaZGWRnGm2yguma+aywTOc2WWH5jLCM0SYj\ngzZZFpbdv12brHBdwv72r7f95cL1yazLzLDI/h4SKZGIyEEpq6yicO125q3cyvxVpcxbtZV12/Ye\n8v6yMmzfQTA7PIBmZdY+0AbzHbKz9h849x0wax1Iaw7oWfvns2oO+AkH7uzwYJ6VsX96fzIIDuxZ\nGfung7KGWfMcnNOJEomIHND6bXuZt2or81ZuZd6qrby/djvlldUA9O/WnnGDujM2tzsDe3TYd/Cu\nSQBZCdNt6ljXJiODjGY6a5boKJGIyD7lldUUrt3GvLClMX/lVtaGrY3srAyO6d+VK08axNjc7owd\n1J3DurSLOWJJBUokIq3Yhu1797U05q0q5b012/a1Nvp1bcdxg7pzdW53xuZ2Y1S/rmRn6YkB+aRI\nE4mZTQTuBzKBx9z97lrrpwD3AmvCRQ+6+2Phuv8FziN41uUl4Ovu7mY2Dvg10B54tmZ5lPUQaQnK\nK6tZtG77vsQxf1Upa0r3AMEF5KP7d+FLJw5ibNhV1aerWhuSnMgSiZllAg8BZwHFwBwzy3f3RbWK\nznD3qbW2/RRwMjA6XPQmcDrwGvAL4FrgLYJEMhF4LqJqiKStjdv37mtpzFu5lffWbKMsbG307dqO\nsbnduerkwYwd1J1R/brQNisz5oglXUXZIhkPFLn7cgAzewqYDNROJHVxoB2QDRjQBthgZn2BLu7+\n73CfvwEuRIlEWrmKqmoWrd3+scSR2NoY1b8LXzix5tpGN/p2bR9zxNKSRJlI+gOrE+aLgRPqKHex\nmZ0GLAO+6e6r3f3fZvYqsI4gkTzo7ovNLC/cT+I++9f15WZ2LUHLhdzc3EZXRiSVbNyxl3krS5m/\nKuimWli8v7XRp0s7xg7qxlUnD+a43O4c3V+tDYlWlImkrnv6al/LmAU86e5lZnYdMB2YYGZHACOA\nAWG5l8JksyeJfQYL3R8BHgHIy8vTNRRJWxVV1Szed22jlPmrt7J6S/BfoU2mMapfVz5/wiDGDurG\n2Nzu9Oum1oY0rygTSTEwMGF+ALA2sYC7b06YfRS4J5z+LPCWu+8EMLPngBOB37I/udS5T5F0V7Kj\nLOyi2sr8laUsXFPK3oqgtXFYl7aMze3Ol04czNhBwZ1U7dqotSHxijKRzAGGmdkQgruyLgc+l1jA\nzPq6+7pwdhKwOJxeBXzFzH5M0LI5Hfi5u68zsx1mdiLwNvAl4IEI6yASqYqqapas27Evccxb9fHW\nxsh+XblifO6+5zb6dW2nJ6sl5USWSNy90symAi8Q3P77hLsXmtkdQIG75wPXm9kkoBLYAkwJN58J\nTADeI+i6et7dZ4Xrvsr+23+fQxfaJY1s2lm2r4tq3qqtvFe8jT0VVQD07hy0Nr4YXhQ/ur9aG5Ie\nrDU8gpGXl+cFBQVxhyGtTGVVNUvW79j3zMa8VVtZuXk3sL+1MTa3m1obkrLMbK675zVUTk+2izSR\nzTvL9rU05q0M7qSqaW3kdG7L2NxufG58LuMGqbUhLYsSicghqKyqZumGHfue2UhsbWRlGCP7deGy\n4wdyXNjiGNC9vVob0mIpkYgkYcuu8n3PbMxbWcq7xaXsLg9aG706Ba2NmovioweotSGtixKJSC2J\nrY354fWNFZt2AftbG5fmqbUhUkOJRFq9ZFoblx0/kLG53Tmmf1faZ6u1IZJIiURalapqZ9mG4E6q\nuSs/3trIzDBG9u3Cf44bsO8NuGptiDRMiURatNLd5ftuvZ23aivvrt7GzrJKAHp1yua43O5cmjeQ\nsbndGD2gm1obIodAiURajMTWRk3yWF6yv7Uxom9nLhrbP3huI7c7A3uotSHSFJRIJG1t213BvNXB\ncLDzVpWyYHXpvtZGz45Ba+OScQP23UnVIVv/3EWioP9Zkla2763gnueW8NbyzXyY0NoY3qcznz2u\n/7434Ob26KDWhkgzUSKRtPKTF5by5DurmDC8NxeN3d/a6NhW/5RF4qL/fZI2Fq/bzu/eWskXTxzE\n7ZOPjjscEQllxB2ASDLcnWn5hXRt34ZvnnVk3OGISAIlEkkLTy9cxzsrtnDjfxxFtw7ZcYcjIgmU\nSCTl7S6v5K5nFzOqXxcuPz437nBEpBZdI5GU93+vfsi6bXt54IrjyMzQnVgiqUYtEklpKzfv4pE3\nlnPhmH7kDe4RdzgiUgclEklpP3xmMVmZxi3njog7FBGphxKJpKzXl5Xw0qINfG3CMA7r0i7ucESk\nHkokkpLKK6u5fVYhQ3p15MunDI47HBE5gEgTiZlNNLOlZlZkZjfXsX6KmZWY2YLwc024/MyEZQvM\nbK+ZXRiu+7WZrUhYNybKOkg8pv/rI5aX7OIH54+kbZbeyCuSyiK7a8vMMoGHgLOAYmCOmeW7+6Ja\nRWe4+9TEBe7+KjAm3E8PoAh4MaHIt919ZlSxS7w2bt/L/S9/wIThvTlzeO+4wxGRBkTZIhkPFLn7\ncncvB54CJh/Cfi4BnnP33U0anaSse55fSnllNbeePzLuUEQkCVEmkv7A6oT54nBZbReb2UIzm2lm\nA+tYfznwZK1lPwq3uc/M2tb15WZ2rZkVmFlBSUnJIVVAmt+8VVv587xirj51CEN6dYw7HBFJQpSJ\npK4nx7zW/CxgsLuPBmYD0z+2A7O+wDHACwmLbwGGA8cDPYCb6vpyd3/E3fPcPS8nJ+fQaiDNqrra\nuS2/kMO6tGXqmUfEHY6IJCnKRFIMJLYwBgBrEwu4+2Z3LwtnHwXG1drHpcBf3b0iYZt1HigDfkXQ\nhSYtwJ/mrmZh8TZuOWeEXgsvkkaiTCRzgGFmNsTMsgm6qPITC4QtjhqTgMW19nEFtbq1araxYNSi\nC4H3mzhuicG2PRX87/NLyRvUnclj+sUdjogchMhO+9y90symEnRLZQJPuHuhmd0BFLh7PnC9mU0C\nKoEtwJSa7c1sMEGL5vVau/69meUQdJ0tAK6Lqg7SfH4+exlbdpczfdJ4jWwokmbMvfZli5YnLy/P\nCwoK4g5D6rFsww7Ouf8fXHb8QO767DFxhyMiITOb6+55DZXTk+0SK3fn9lmFdGqbxY1nHxV3OCJy\nCJRIJFbPv7+efxZt5oazj6RHRw1YJZKOlEgkNnvKq/jhM4sZ3qcznxuvAatE0pXusZTYPPzGh6wp\n3cNT155IVqbOaUTSlf73SixWb9nNL177kPNH9+XEoT3jDkdEGkGJRGJx17OLyTDjuxqwSiTtKZFI\ns/tn0Saee389/3Pm4fTr1j7ucESkkZRIpFlVVFVzW34hA3u055pTh8Ydjog0ASUSaVa//fdKPti4\nk1vPG0m7NhqwSqQlUCKRZrNpZxn3zV7GaUfmcNbIw+IOR0SaiBKJNJt7n1/KnvIqfnD+SL1PS6QF\nUSKRZvHu6lL+OHc1Xz5lCEf07hR3OCLShJRIJHLV1c5tswrp2bEtX5ugAatEWholEoncX+avYf6q\nUm4+Zzid27WJOxwRaWJKJBKpHXsruPu5JRyX242LjusfdzgiEgG9a0si9cArRWzeVcbjV+aRkaEL\n7CItkVokEpmijTt54s0VXDpuIMcO7BZ3OCISESUSiUTNgFXtszP59kQNWCXSkimRSCRmL97IPz7Y\nxDc/cyS9OrWNOxwRiZASiTS5vRVV3Pn0Iob17sQXTxoUdzgiErFIE4mZTTSzpWZWZGY317F+ipmV\nmNmC8HNNuPzMhGULzGyvmV0YrhtiZm+b2QdmNsPMND5rinnsH8tZtWU3t00aRRsNWCXS4kX2v9zM\nMoGHgHOAkcAVZjayjqIz3H1M+HkMwN1frVkGTAB2Ay+G5e8B7nP3YcBW4Oqo6iAHb23pHh569UPO\nOboPJx/RK+5wRKQZRHm6OB4ocvfl7l4OPAVMPoT9XAI85+67LXhB0wRgZrhuOnBhk0QrTeKuZxdT\n7a4Bq0RakSgTSX9gdcJ8cbistovNbKGZzTSzgXWsvxx4MpzuCZS6e2UD+8TMrjWzAjMrKCkpObQa\nyEF5a/lmnl64jq+ecTgDe3SIOxwRaSZRJpK6nj7zWvOzgMHuPhqYTdDC2L8Ds77AMcALB7HPYKH7\nI+6e5+55OTk5BxW4HLzKcMCq/t3ac93ph8cdjog0oygTSTGQ2MIYAKxNLODum929LJx9FBhXax+X\nAn9194pwfhPQzcxqnsj/xD4lHn94ZxVL1u/g++eN0IBVIq1MlIlkDjAsvMsqm6CLKj+xQNjiqDEJ\nWFxrH1ewv1sLd3fgVYLrJgBXAn9v4rjlIG3ZVc5PX1zGyUf0ZOLRfeIOR0SaWWSJJLyOMZWgW2ox\n8Ed3LzSzO8xsUljsejMrNLN3geuBKTXbm9lgghbN67V2fRPwLTMrIrhm8nhUdZDk/PTFpewsq2Ta\nBaM0YJVIK2TBSX7LlpeX5wUFBXGH0SK9v2YbFzz4JlM+NZhpF4yKOxwRaUJmNtfd8xoql1SLxMxO\nNrOO4fQXzOxnZqZHlls5d+e2/EJ6dMjmG585Mu5wRCQmyXZt/QLYbWbHAt8BVgK/iSwqSQv5766l\nYOVWvjPxKLq214BVIq1VsomkMrzQPRm4393vBzpHF5akul1lldz17GJGD+jKf46r6/EfEWktkh3Y\naoeZ3QJ8ATgtfP2JTkFbsQdfLWLD9jJ+8YVxGrBKpJVLtkVyGVAGXO3u6wmeJr83sqgkpa3YtIvH\n/7GCi8cOYGxu97jDEZGYJd0iIejSqjKzI4HhJDzfIa3LnU8vIjsrg5vO0YBVIpJ8i+QNoK2Z9Qde\nBq4Cfh1VUJK6XlmygVeWbOTrnx5G787t4g5HRFJAsonE3H03cBHwgLt/FtBDA61MWWUVdz69mKE5\nHbnyU4PjDkdEUkTSicTMTgI+DzwTLtMLlVqZJ978iBWbdjHtglFkZ2nAKhEJJHs0+AZwC8ELFAvN\nbCjBO6+kldiwfS8PvPIBZ408jNOP1NuURWS/pC62u/vrwOtm1tnMOrn7coJ3Y0krcfdzS6isdm49\nr65BLkWkNUv2FSnHmNl84H1gkZnNNTNdI2klCj7awl/nr+HaU4eS21MDVonIxyXbtfUw8C13H+Tu\nucANBOOHSAtXVe1Myy+kb9d2/PeZGrBKRD4p2UTS0d33XRNx99eAjpFEJCllxpzVFK7dznfPHUGH\n7GQfOxKR1iTZI8NyM7sV+G04/wVgRTQhSaoo3V3OvS8s4YQhPTh/dN+GNxCRVinZFsmXgRzgL8Bf\nw+mrogpKUsN9Ly1j254KbpukAatEpH7J3rW1Fd2l1aosWb+d3761ki+cOIgRfbvEHY6IpLADJhIz\nmwXUO4Siu0+qb52kL3dn2t8L6dq+Dd86SwNWiciBNdQi+UmzRCEp5Zn31vH2ii386LNH061Ddtzh\niEiKO2AiCR9EPGRmNhG4n+B1Ko+5+9211k8heB39mnDRg+7+WLguF3gMGEjQKjrX3T8ys18DpwPb\nwm2muPuCxsQp++0ur+SuZxYzsm8XLj8+N+5wRCQNJHWNxMze45NdXNuAAuCH7r65jm0ygYeAs4Bi\nYI6Z5bv7olpFZ7j71Dq+9jfAj9z9JTPrBFQnrPu2u89MJnY5OL947UPWbtvL/VccR6YGrBKRJCR7\n++9zQBXwh3D+csAIksmvgQvq2GY8UBS+TgUze4pgqN7aieQTzGwkkOXuLwG4+84k45RGWLV5Nw+/\nsZwLx/Tj+ME94g5HRNJEsrf/nuzut7j7e+Hne8Dp7n4PMLiebfoDqxPmi8NltV1sZgvNbKaZ1Qz+\nfSRQamZ/MbP5ZnZv2MKp8aNwm/vMrG2SdZAG3PnMIrIyjJvPGRF3KCKSRpJNJJ3M7ISaGTMbD3QK\nZyvr2aaufpHa3WOzgMHuPhqYDUwPl2cBpwI3AscDQ4Ep4bpbCEZoPB7oAdxU55ebXWtmBWZWUFJS\nUn/NBIDXl5Xw0qINTJ1wBH26asAqEUlesonkGuAxM1thZh8RXAS/xsw6Aj+uZ5tiggvlNQYAaxML\nuPtmdy8LZx8FxiVsO9/dl7t7JfA3YGy4zToPlAG/IuhC+wR3f8Td89w9LydHrz0/kPLKam6fVcjg\nnh24+pQhcYcjImkm2QcS5wDHmFlXgtESSxNW/7GezeYAw8xsCMFdWZcDn0ssYGZ93X1dODsJWJyw\nbXczy3H3EmACwYX9fdtY8Kj1hQRvJJZGmP6vj1hesosnpuTRNkvjlYnIwUn2rq2uwDTgtHD+deAO\nd99W3zbuXmlmU4EXCG7/fSIcFOsOoMDd84HrzWwSQffYFsLuK3evMrMbgZfDhDGX/W8b/r2Z5RB0\nnS0ArjvIOkuCjTv2cv/LH3DmUTlMGH5Y3OGISBoy93ofXN9fyOzPBGf+Ndcwvggc6+4XRRhbk8nL\ny/OCgoK4w0hJN/7pXf6+YA0vfvN0hvTSC51FZD8zm+vueQ2VS/b238Pd/eKE+dvNTA8Bprl5q7Yy\nc24x151+uJKIiByyZC+27zGzU2pmzOxkYE80IUlzqK52bssvpHfntkydcETc4YhIGku2RfJVYHrN\nxXYSrmdIepo5t5iFxdv4+WVj6NRWA1aJyKFL9q6tBcCxZtYlnN8eaVQSqW17Krjn+SWMG9SdyWP6\nxR2OiKS5hl4j/616lgPg7j+LICaJ2P2zP2DL7nKmTxqvAatEpNEaapF0bpYopNl8sGEH0//9EVeM\nz+Xo/l3jDkdEWoCGXiN/e3MFItFzd26bVUjH7ExuPPuouMMRkRYi2bu29jGzeVEEItF7oXA9/yza\nzA1nH0WPjhqwSkSaxkEnEup+GaOkuL0VVdz59GKG9+nM50/QgFUi0nQO5b7PZ5o8ConcL1//kDWl\ne3jyKyeSlXko5w8iInU76COKu38/ikAkOsVbd/OL1z7kvNF9OenwnnGHIyItTLIvbdxB/UPt3lAz\nCqKkprueXYwZfO9cDVglIk0v2a6tnxGMJfIHgmsklwN9gKXAE8AZUQQnjffPok08+956bjjrSPp1\nax93OCLSAiXbtTXR3R929x3uvt3dHwHOdfcZQPcI45NGqKgKBqwa2KM9XzltaNzhiEgLlWwiqTaz\nS80sI/xcmrCu4ffQSyx+99ZKlm3Yya3njaRdGw1YJSLRSDaRfJ5gDJKNwIZw+gtm1h6YGlFs0gib\ndpbxs5eWceqwXpw1UgNWiUh0kn1p43LggnpWv9l04UhT+ckLS9lTXsW0C0bpfVoiEqmkWiRmdqSZ\nvWxm74fzo81MtwGnqIXFpcwoWM1VJw/miN6d4g5HRFq4ZLu2HgVuASoA3H0hwZ1bkmKqq51p+YX0\n7NiW6z89LO5wRKQVSDaRdHD3d2otq2zqYKTx/jp/DfNXlXLzOcPp3K5N3OGISCuQbCLZZGaHE96h\nZWaXAOsa2sjMJprZUjMrMrOb61g/xcxKzGxB+LkmYV2umb1oZovNbJGZDQ6XDzGzt83sAzObYWZ6\n+2Box94K7n5+CWMGduOi4/rHHY6ItBLJJpL/AR4GhpvZGuAbwHUH2sDMMoGHgHOAkcAVZjayjqIz\n3H1M+HksYflvgHvdfQQwnuCOMYB7gPvcfRiwFbg6yTq0eA+8UsSmnWXcPmkUGRm6wC4izSPZRLIG\n+BXwI+Ap4CXgyga2GQ8Uuftydy8Pt5uczJeFCSfL3V8CcPed7r7bgtuPJgAzw6LTgQuTrEOLVrRx\nJ0+8uYJLxw3k2IHd4g5HRFqRZBPJ3wlu/60geFXKTmBXA9v0B1YnzBeHy2q72MwWmtlMMxsYLjsS\nKDWzv5jZfDO7N2zh9ARK3b2ygX22Ku7OHU8von2bTL49UQNWiUjzSvZdWwPcfeJB7ruuvpXaT8HP\nAp509zIzu46ghTEhjOtU4DhgFTADmALkJ7HP4MvNrgWuBcjNbdnjb8xevJE3lpVw6/kj6dWpbdzh\niEgrk2yL5F9mdsxB7rsYGJgwP4CgNbOPu29297Jw9lFgXMK288NusUrgb8BYYBPQzcyy6ttnwr4f\ncfc8d8/Lyck5yNDTRzBg1SKG9e7El04aFHc4ItIKJZtITgHmhndgLTSz98xsYQPbzAGGhXdZZRM8\nd/KxFoWZ9U2YnQQsTti2u5nVZIAJwCJ3d+BV4JJw+ZUE3W6t1uNvrmDVlt1Mu2AUbTRglYjEINmu\nrXMOdsfuXmlmU4EXgEzgCXcvNLM7gAJ3zweuN7NJBM+kbCHovsLdq8zsRuDl8AL7XIIWC8BNwFNm\n9kNgPvD4wcbWUqwt3cODrxQxcVQfThnWK+5wRKSVsuAkv2XLy8vzgoKCuMNocl97cj4vFq5n9rdO\nZ2CPDnGHIyItjJnNdfe8hsqpLyRNvb18M7PeXct1px+uJCIisVIiSUOVVdVMyy+kf7f2XHf64XGH\nIyKtnBJJGnrynVUsWb+D7583gvbZGrBKROKlRJJmtu4q5ycvLuNTh/dk4tF94g5HRESJJN385MWl\n7Cyr5LZJGrBKRFKDEkkaeX/NNv7wziq+dNIgjjysc9zhiIgASiRpw925fVYh3Ttk843PHBl3OCIi\n+yiRpIn8d9cy56OtfOc/jqJrew1YJSKpQ4kkDewqq+SuZxczekBXLs0b2PAGIiLNSIkkDTz0ahEb\ntpcx7QINWCUiqUeJJMV9tGkXj/1jBReN7c+4Qd3jDkdE5BOUSFLcnU8vIjsrg5snDo87FBGROimR\npLBXl2zk5SUbuf7TR9C7S7u4wxERqZMSSYoqq6zijqcXMTSnI1M+NSTucERE6qVEkqJ+9c+PWLFp\nFz84fyTZWfqZRCR16QiVgjZs38sDL3/AZ0YcxhlH9Y47HBGRA1IiSUF3P7eEimrn1vNHxB2KiEiD\nlEhSTMFHW/jr/DVce+pQBvXsGHc4IiINUiJJIVXVzm2zCunTpR3/faYGrBKR9KBEkkJmzFnN+2u2\n893zRtAhOyvucEREkhJpIjGziWa21MyKzOzmOtZPMbMSM1sQfq5JWFeVsDw/YfmvzWxFwroxUdah\nuWzbXcG9Lyxh/JAeXDC6b9zhiIgkLbLTXjPLBB4CzgKKgTlmlu/ui2oVneHuU+vYxR53ry9JfNvd\nZzZhuLG7b/Yytu2p4LYLNGCViKSXKFsk44Eid1/u7uXAU8DkCL8vbS1Zv53fvrWSz58wiJH9usQd\njojIQYkykfQHVifMF4fLarvYzBaa2UwzS3xHejszKzCzt8zswlrb/Cjc5j4za9vUgTcnd+e2/EI6\nt8vihrM1YJWIpJ8oE0ld/TNea34WMNjdRwOzgekJ63LdPQ/4HPBzM6u5jekWYDhwPNADuKnOLze7\nNkxEBSUlJY2oRrSefW89by3fwo1nH0W3DtlxhyMictCiTCTFQGILYwCwNrGAu29297Jw9lFgXMK6\nteGfy4HXgOPC+XUeKAN+RdCF9gnu/oi757l7Xk5OTtPUqIntLq/kR88sYmTfLlwxPjfucEREDkmU\niWQOMMzMhphZNnA5kJ9YwMwSb0+aBCwOl3ev6bIys17AycCixG0suCJ9IfB+hHWI1C9f+5C12/Zy\n++RRZGrAKhFJU5HdteXulWY2FXgByASecPdCM7sDKHD3fOB6M5sEVAJbgCnh5iOAh82smiDZ3Z1w\nt9fvzSyHoOtsAXBdVHWI0uotu/nlG8uZPKYfxw/uEXc4IiKHzNxrX7ZoefLy8rygoCDuMD7m2t8U\n8GbRJl654Qz6dNVYIyKSesxsbnit+oD0ZHsM3lhWwouLNjB1whFKIiKS9pRImllFVTW3zypkUM8O\nXH2KBqwSkfSnRNLMpv/rIz4sCQasapuVGXc4IiKNpkTSjDbu2MvPZ3/AmUfl8OkRh8UdjohIk1Ai\naUb3Pr+Ussoqbj1/ZNyhiIg0GSWSZjJ/1Vb+NLeYL58yhKE5neIOR0SkySiRNIPq6uB9Wr07t+Vr\nE4bFHY6ISJNSImkGM+cV827xNm45dzid2mrAKhFpWZRIIrZ9bwX/+/wSxg3qzoVj6nr5sYhIetPp\nccTun/0Bm3eV8+urxmvAKhFpkdQiidAHG3Yw/V8fcfnxuRzdv2vc4YiIREKJJCLuzu2zFtEhO5Mb\nNWCViLRgSiQReaFwA28WbeKGs4+iZ6e0HsRRROSAlEgisLeiih8+s4ijDuvM50/QgFUi0rLpYnsE\nHn59OcVb9/DkV04kK1O5WkRaNh3lmljx1t3832tFnDe6Lycd3jPucEREIqdE0sR+/OwSzOC7546I\nOxQRkWahRNKE/lW0iWfeW8f/nHEE/bu1jzscEZFmoUTSRCqrqrltViEDe7TnK6cNjTscEZFmo0TS\nRH731kqWbdjJ988bSbs2GrBKRFqPSBOJmU00s6VmVmRmN9exfoqZlZjZgvBzTcK6qoTl+QnLh5jZ\n22b2gZnNMLPsKOuQjM07y/iLftK3AAAJQUlEQVTZS8s4dVgvzh6pAatEpHWJLJGYWSbwEHAOMBK4\nwszqGtFphruPCT+PJSzfk7B8UsLye4D73H0YsBW4Oqo6JOsnLy5ld3kV0y4YqfdpiUirE2WLZDxQ\n5O7L3b0ceAqY3JgdWnCUngDMDBdNBy5sVJSNtLC4lKfmrGbKpwZzRO/OcYYiIhKLKBNJf2B1wnxx\nuKy2i81soZnNNLOBCcvbmVmBmb1lZjXJoidQ6u6VDeyzWdQMWNWzY1u+/hkNWCUirVOUiaSuPh6v\nNT8LGOzuo4HZBC2MGrnungd8Dvi5mR2e5D6DLze7NkxEBSUlJQcffRL+tmAN81aVctPEo+jcrk0k\n3yEikuqiTCTFQGILYwCwNrGAu29297Jw9lFgXMK6teGfy4HXgOOATUA3M6t5tcsn9pmw/SPunufu\neTk5OY2vTS079lbw4+eWMGZgNy4eO6DJ9y8iki6iTCRzgGHhXVbZwOVAfmIBM+ubMDsJWBwu725m\nbcPpXsDJwCJ3d+BV4JJwmyuBv0dYh3o9+EoRJTvKuH3SKDIydIFdRFqvyF7a6O6VZjYVeAHIBJ5w\n90IzuwMocPd84HozmwRUAluAKeHmI4CHzayaINnd7e6LwnU3AU+Z2Q+B+cDjUdWhPh+W7OSJf67g\n0rwBHDuwW3N/vYhISrHgJL9ly8vL84KCgibZl7sz5VdzmLdyK6/ceAY5nTXWiIi0TGY2N7xWfUB6\nsv0gvbx4I68vK+EbZx2pJCIighLJQdlbUcUdTy/iiN6d+NJJg+IOR0QkJWhgq4Pw+JsrWLVlN7+7\n+gTaaMAqERFALZKkrdu2hwdfKWLiqD6cMqxX3OGIiKQMJZIk/fjZJVS7873zNGCViEgiJZIkvL18\nM/nvruW/Tj+cgT06xB2OiEhKUSJpQGVVNdPyC+nfrT1fPf3wuMMREUk5SiQNeHLOapas38H3zhtB\n+2wNWCUiUpsSyQFs3VXOT19cyklDe3LO0X3iDkdEJCUpkRzAT19ayo69ldw2aZQGrBIRqYcSyQEM\n7N6B/zptKEf10YBVIiL10QOJB/BfurguItIgtUhERKRRlEhERKRRlEhERKRRlEhERKRRlEhERKRR\nlEhERKRRlEhERKRRlEhERKRRzN3jjiFyZlYCrDzEzXsBm5ownDi1lLq0lHqA6pKqWkpdGluPQe6e\n01ChVpFIGsPMCtw9L+44mkJLqUtLqQeoLqmqpdSlueqhri0REWkUJRIREWkUJZKGPRJ3AE2opdSl\npdQDVJdU1VLq0iz10DUSERFpFLVIRESkUZRIQmY20cyWmlmRmd1cx/q2ZjYjXP+2mQ1u/igblkQ9\npphZiZktCD/XxBFnMszsCTPbaGbv17PezOz/hXVdaGZjmzvGZCRRjzPMbFvCb/KD5o4xWWY20Mxe\nNbPFZlZoZl+vo0zK/y5J1iMtfhcza2dm75jZu2Fdbq+jTLTHL3dv9R8gE/gQGApkA+8CI2uV+W/g\nl+H05cCMuOM+xHpMAR6MO9Yk63MaMBZ4v5715wLPAQacCLwdd8yHWI8zgKfjjjPJuvQFxobTnYFl\ndfwbS/nfJcl6pMXvEv49dwqn2wBvAyfWKhPp8UstksB4oMjdl7t7OfAUMLlWmcnA9HB6JvBpS72B\n3JOpR9pw9zeALQcoMhn4jQfeArqZWd/miS55SdQjbbj7OnefF07vABYD/WsVS/nfJcl6pIXw73ln\nONsm/NS++B3p8UuJJNAfWJ0wX8wn/1HtK+PulcA2oGezRJe8ZOoBcHHY5TDTzAY2T2iRSLa+6eCk\nsGviOTMbFXcwyQi7R44jOANOlFa/ywHqAWnyu5hZppktADYCL7l7vb9JFMcvJZJAXZm5dkZPpkzc\nkolxFjDY3UcDs9l/lpKO0uE3ScY8gldRHAs8APwt5ngaZGadgD8D33D37bVX17FJSv4uDdQjbX4X\nd69y9zHAAGC8mR1dq0ikv4kSSaAYSDwzHwCsra+MmWUBXUm97ooG6+Hum929LJx9FBjXTLFFIZnf\nLeW5+/aargl3fxZoY2a9Yg6rXmbWhuDg+3t3/0sdRdLid2moHun2uwC4eynwGjCx1qpIj19KJIE5\nwDAzG2Jm2QQXo/JrlckHrgynLwFe8fDKVQppsB61+qonEfQNp6t84EvhXUInAtvcfV3cQR0sM+tT\n019tZuMJ/l9ujjequoVxPg4sdvef1VMs5X+XZOqRLr+LmeWYWbdwuj3wGWBJrWKRHr+ymmpH6czd\nK81sKvACwZ1PT7h7oZndARS4ez7BP7rfmlkRQSa/PL6I65ZkPa43s0lAJUE9psQWcAPM7EmCO2d6\nmVkxMI3gQiLu/kvgWYI7hIqA3cBV8UR6YEnU4xLgq2ZWCewBLk/Bk5QaJwNfBN4L++QBvgvkQlr9\nLsnUI11+l77AdDPLJEh2f3T3p5vz+KUn20VEpFHUtSUiIo2iRCIiIo2iRCIiIo2iRCIiIo2iRCIi\nIo2iRCLSBMxsZ8OlDrj9TDMbGk53MrOHzezD8G2ub5jZCWaWHU7rtn1JKUokIjEL3+GU6e7Lw0WP\nEdzrP8zdRxE869MrfBHny8BlsQQqUg8lEpEmFD7Nfa+ZvW9m75nZZeHyDDP7v7CF8bSZPWtml4Sb\nfR74e1jucOAE4PvuXg0Qvs35mbDs38LyIilDTWSRpnURMAY4FugFzDGzNwiepB4MHAP0Jng1zRPh\nNicDT4bTo4AF7l5Vz/7fB46PJHKRQ6QWiUjTOgV4Mnwb6wbgdYID/ynAn9y92t3XA68mbNMXKElm\n52GCKTezzk0ct8ghUyIRaVr1DRZ0oEGE9gDtwulC4FgzO9D/zbbA3kOITSQSSiQiTesN4LJwoKEc\ngmF23wHeJBhQLMPMDiN4iWONxcARAO7+IVAA3J7w5tlhZjY5nO4JlLh7RXNVSKQhSiQiTeuvwELg\nXeAV4DthV9afCcaEeB94mGA0vm3hNs/w8cRyDdAHKDKz9wjGjakZz+NMgrfriqQMvf1XpJmYWSd3\n3xm2Kt4BTnb39eEYEq+G8/VdZK/Zx1+AW9x9aTOELJIU3bUl0nyeDgcgygbuDFsquPseM5tGMK72\nqvo2Dgcr+5uSiKQatUhERKRRdI1EREQaRYlEREQaRYlEREQaRYlEREQaRYlEREQaRYlEREQa5f8D\n5BvL75Kj3dIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xee30b38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "\n",
    "n_classes = 5\n",
    "\n",
    "array = lrcv_L1.scores_\n",
    "\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "   # print(np.mean(tuple(array.values())[0][j],axis = 0))\n",
    "    print tuple(array.values())[0][j]\n",
    "    scores[j][:] = tuple(array.values())[0][j]\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.3310358 ,  1.06732454,  0.        ,  0.0461603 , -0.3676885 ,\n",
       "         0.59140289,  0.29851319,  0.        ]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### l2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(train_X, train_y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.62320614, -0.65308667, -0.65747315, -0.65793249],\n",
       "        [-0.62338804, -0.64838829, -0.65166731, -0.65200561],\n",
       "        [-0.48597717, -0.49599931, -0.4975406 , -0.49770242],\n",
       "        [-0.58711603, -0.6122243 , -0.61565459, -0.61601044],\n",
       "        [-0.47140616, -0.48266391, -0.48428644, -0.48445543]])}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.62320614 -0.65308667 -0.65747315 -0.65793249]\n",
      "[-0.62338804 -0.64838829 -0.65166731 -0.65200561]\n",
      "[-0.48597717 -0.49599931 -0.4975406  -0.49770242]\n",
      "[-0.58711603 -0.6122243  -0.61565459 -0.61601044]\n",
      "[-0.47140616 -0.48266391 -0.48428644 -0.48445543]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xmc1XW9x/HXZzb2bWBAZJtBdlRQ\nhtFck9JwCSy9ZotJ5fXWvWZdb4taiVLWNSu7lbdc0qxHmWWlLO5GlF4TBkVlGJCRdVgHhn2Y/XP/\n+P2g0zgwB8458ztn5v18PM6D81vP+8uB3+f8ft/fYu6OiIjI8cqKOoCIiGQ2FRIREUmIComIiCRE\nhURERBKiQiIiIglRIRERkYSokIiISEJUSEREJCEqJCIikpCcqAO0hwEDBnhhYWHUMUREMsrSpUt3\nuHtBW/N1ikJSWFhIaWlp1DFERDKKma2PZz4d2hIRkYSokIiISEJUSEREJCEqJCIikhAVEhERSYgK\niYiIJESFREREEtIpriMREUl37k5Ts1Pf1ExDY/Bn8L6ZhvB9fWMzDU0eDDeG0w+Pb6a+yQ+/bwin\nf+rsIvJ75KU0uwqJiHQKzc1OQ3McG+NGP+IGvL6xKfizxTINTU7d4fctN+xNR/k8p6GxmbpwnHvy\n2z1z8okqJCKS/lL1a/of62uioTFYtq7Feg9/XsyGvLUNe2Nz8rfSOVlGbnYWeTlZwZ/Zdvj9ofF5\n2Vl0zc2id9ecYHxOFl3C6bk5Rl52dvhnMG9uTuyyLdefFTMcTOty6PPCz8o7vN4ssrMMM0t6u9/1\n95DyTxCRDqexqZmyzXtZsq6aJeuqKV23i50H6pP+OYc2xLEbzkPjYjeovXJz/rFBjWPDnptt5OVk\nh3/GbKTD9bVcV2sb8LzsLLKyUr+RzgQqJCLSpoP1Tby+YReLw6Lx2oZd1NQ3ATAsvxvnjy2gsH+P\nf9qAx27Q/+mXc+zGO6fFBjz8dZ6bnUVOO/2alsSpkIjIu+w6UB/saazfxeK11SzftIfGZscMxg7q\nxZVThlJcmE9JYT4n9OkadVyJmAqJiFC5q4bSdcEex5K11azevh+AvOwsTh3ah389byRTC/sxZXg+\nfbrnRpxW0o0KiUgn09zsVFTtZ/HaoH9jydpqNu+pBaBnlxymjOjHzMknMrUwn0nD+tI1NzvixJLu\nVEhEOrj6xmaWb97DkrBwlK7fxe6aBgAKenWhpDCffy3sx9TCfMYP7k22OpDlGKmQiHQwB+oaeW3D\nLpasrWbxumqWbdxNbUMzAEUDenDh+EFMLQr6N0b0764ObUmYColIhtuxv47SddUsXruL0vXVlG3e\nS1Ozk2UwfnBvrp46nJKifIoL+zGwlzrGJflSWkjMbDrwP0A28KC7/3eL6bOAu4FN4aifuPuD4bTv\nApcS3A/seeAL7u5mNgX4BdANeOrQ+FS2QyRduDsbqw8e7hRfsr6aNVUHgOCai8nD+vK5809ialE+\npw/vS6+u6hiX1EtZITGzbOBe4EKgElhiZnPdfUWLWR9z9xtaLHsWcDZwajjqJeB84C/AT4Hrgb8T\nFJLpwNMpaoZIpJqanVVb97FkXXV4DUc12/bWAdC7aw7Fhfn8y5RhlBT14+QhfeiSo45xaX+p3CMp\nASrcfQ2Amf0WmAm0LCStcaArkAcYkAtsM7PBQG93fyVc5y+By1EhkQ6irrGJNyv3sHhtUDRK1+9i\nX20jACf07kpJUX9KCvsxtSifMQN76cpqSQupLCRDgI0xw5XAGa3Md4WZnQe8Dfynu29091fMbCGw\nhaCQ/MTdy82sOFxP7DqHpCa+SOrtrW1g6fpdlK6rZsnaXSyr3E19Y9AxflJBDy47dTBTC/OZWpjP\n0H7d1DEuaSmVhaS1f/Et+zLmAY+6e52ZfRZ4BJhmZqOA8cDQcL7nw2JzMI51Bh9udj3BITCGDx9+\nHPFFkm/73lqWrNsVHKpaW83KrXtpdsjOMk4+sTefPHMEU4vyKR7Rj/49u0QdVyQuqSwklcCwmOGh\nwObYGdx9Z8zgA8Bd4fsPAX939/0AZvY0cCbwK/5RXFpdZ8y67wfuByguLlZnvLQ7d2fdzprDp+Eu\nWVfN+p01AHTLzea04X35/LTRlBTlM3lYX3p00UmUkplS+S93CTDazIoIzsq6GvhY7AxmNtjdt4SD\nM4Dy8P0G4F/N7DsEezbnAz909y1mts/MzgReBT4J/DiFbRCJW1OzU75l7z+uGF+3ix37g47xft1z\nKS7M5xNnjKC4MOgYz83WA0qlY0hZIXH3RjO7AXiW4PTfh9y9zMzmAKXuPhe40cxmAI1ANTArXPxx\nYBrwFsGhq2fcfV447XP84/Tfp1FHu0SktqGJZRt3H97jeH3DbvbXBR3jQ/p249zRA8L+jX6cVNBT\nHePSYVlnuASjuLjYS0tLo44hGW5PTQOl66sPX8Px1qY9NDQF/3/GDurF1KJ+hzvGT+zbLeK0Iokz\ns6XuXtzWfDooK3IEW/YcjLmx4S5WbdsHQG62ccqQPnz6nCKmjgiuGO/bPbWPMhVJZyokIgQd4+9U\n7Wfx2l2Hn/pXuSs4SbBHXjanj+jHpeGpuJOH9aVbni78EzlEhUQ6pYZDj4qNuSNudfio2AE98yge\nkc+nzi6ipDCf8YN7kaOOcZEjUiGRTqGmvpHXN+w+vLfx2vrdHGwIHhU7on93Lhg7kJKwj6NoQA9d\n+CdyDFRIpEOqPvSo2HXVLF63i7KYR8WOP6E3VxUPZWpR0DE+qLfuiCuSCBUSyXjuTuWug8EZVWEf\nR0XMo2InDevD9eeNZGpRPlNG9KO37ogrklQqJJJxmpudt7fvC241EvZxbAkfFdurSw5TCvvxodOG\nUFKUzylD+uhRsSIppkIiGaV8y14++dBiqvYFV4wP7NXl8NP+phbmM/aEXnpUrEg7UyGRjOHufP2J\n5TQ1O9/7l0mUFOYzLF93xBWJmgqJZIw/vb6Jpet38d0rTuXKKUPbXkBE2oVOjpeMsK+2gW8/tZJJ\nw/qqiIikGe2RSEb40Yur2Xmgjp9fW6ybH4qkGe2RSNpbvW0fD7+8jo8UD2PSsL5RxxGRFlRIJK25\nO7fPK6N7XjZf/sDYqOOISCtUSCStPb18Ky9X7ORLHxirR8+KpCkVEklbNfWNfGv+CsYP7s3HSoZH\nHUdEjkCFRNLWT//yDpv31DJn5kTdfVckjel/p6SldTsOcN+iNVw++USmFuZHHUdEjkKFRNLSN+ev\nIDfbuOWS8VFHEZE2qJBI2nmxfBsvrtzOF94/Wrd4F8kAKiSSVmobmpgzfwUnFfRg1llFUccRkTjo\nynZJKw/+bQ3rd9bwq8+UkJej3zkimUD/UyVtbNp9kJ8srGD6xBM4d3RB1HFEJE4qJJI2vr2gHHf4\n+mXqYBfJJCokkhZertjBgre28B8XjGJov+5RxxGRY6BCIpFraGpm9twyhud35/rzRkYdR0SOkQqJ\nRO6R/1tHxfb93HbZBD1fXSQDqZBIpLbvq+WHL6zmgrEFvG/8wKjjiMhxUCGRSP330yupb2zmtg9O\n1LPXRTKUColEpnRdNX98bRPXnVtE0YAeUccRkeOkQiKRaGp2bnuyjMF9unLDtFFRxxGRBKiQSCR+\ns3gDK7bs5WuXjqd7nm6wIJLJVEik3e06UM/3n1vFe0b259JTBkcdR0QSpEIi7e7u51axr7aRO2aq\ng12kI0hpITGz6Wa2yswqzOzmVqbPMrMqM1sWvq4Lx18QM26ZmdWa2eXhtF+Y2dqYaZNT2QZJrrcq\n9/Do4g1c+55CxgzqFXUcEUmClB2cNrNs4F7gQqASWGJmc919RYtZH3P3G2JHuPtCYHK4nnygAngu\nZpYvu/vjqcouqdHc7Nw2dzn9e+TxxQtHRx1HRJIklXskJUCFu69x93rgt8DM41jPlcDT7l6T1HTS\n7v7wWiWvb9jNzRePp3fX3KjjiEiSpLKQDAE2xgxXhuNausLM3jSzx81sWCvTrwYebTHuznCZe8ys\nS2sfbmbXm1mpmZVWVVUdVwMkefbWNnDXMys5fXhfPnxaa/8MRCRTpbKQtNaL6i2G5wGF7n4q8ALw\nyD+twGwwcArwbMzoW4BxwFQgH/hqax/u7ve7e7G7FxcU6NkWUfvh86vZeaCeOTNPJitLHewiHUkq\nC0klELuHMRTYHDuDu+9097pw8AFgSot1XAX8yd0bYpbZ4oE64GGCQ2iSxlZt3ccjr6zjoyXDOXlI\nn6jjiEiSpbKQLAFGm1mRmeURHKKaGztDuMdxyAygvMU6PkqLw1qHlrHgvNHLgeVJzi1J5O7Mnruc\nXl1z+PJFY6OOIyIpkLKztty90cxuIDgslQ085O5lZjYHKHX3ucCNZjYDaASqgVmHljezQoI9mkUt\nVv1rMysgOHS2DPhsqtogiZv/5hb+vqaab11+Mv165EUdR0RSwNxbdlt0PMXFxV5aWhp1jE7nQF0j\n7/v+Ivr3zGPuDeeQrb4RkYxiZkvdvbit+XSTI0mZexdWsHVvLfd+/DQVEZEOTLdIkZRYU7WfB/62\nhg+fPoQpI/KjjiMiKaRCIknn7twxbwVdcrK5+eJxUccRkRRTIZGke6F8O4veruKL7x/NwF5do44j\nIimmQiJJVdvQxJz5ZYwe2JNrzyqMOo6ItAN1tktS3bdoDRurD/Kb684gN1u/U0Q6A/1Pl6TZWF3D\n//6lgktPHcxZowZEHUdE2okKiSTNnQvKyTLja5eMjzqKiLQjFRJJir++XcUzZVu5YdooTuzbLeo4\nItKOVEgkYfWNzdw+r4zC/t257tyiqOOISDtTZ7sk7OGX17Km6gAPz5pKl5zsqOOISDuLa4/EzM42\nsx7h+0+Y2Q/MbERqo0km2La3lh+9uJr3jx/IBeMGRh1HRCIQ76GtnwI1ZjYJ+AqwHvhlylJJxvjO\nU+U0NDvfuGxC1FFEJCLxFpJGD24TPBP4H3f/H6BX6mJJJnh1zU6eWLaZfztvJCP694g6johEJN4+\nkn1mdgvwCeA8M8sGclMXS9JdY1Mzs+eWMaRvN/79vaOijiMiEYp3j+QjQB3wGXffCgwB7k5ZKkl7\nv351Ayu37uPrl46nW5462EU6s7j3SAgOaTWZ2RhgHC0egSudx879dXz/uVWcM2oA008+Ieo4IhKx\nePdI/gp0MbMhwIvAp4BfpCqUpLe7n11FTX0Tt8+YgJkeWCXS2cVbSMzda4APAz929w8BE1MXS9LV\nso27eax0I586u5BRA3W+hYgcQyExs/cAHwcWhON0YLyTaW52Zj+5nAE9u3Dj+0ZHHUdE0kS8heSL\nwC3An9y9zMxGAgtTF0vS0e+XbuSNyj3cesk4enXVSXsiEoirs93dFwGLzKyXmfV09zXAjamNJulk\nT00Ddz2ziqmF/bh88pCo44hIGon3FimnmNnrwHJghZktNTP1kXQi97zwNrtr6rl9xkR1sIvIP4n3\n0NZ9wE3uPsLdhwP/BTyQuliSTsq37OWXr6zj42eMYOKJfaKOIyJpJt5C0sPdD/eJuPtfAN0ToxNw\nd2Y/WUafbrn810Vjoo4jImko3kKyxsy+YWaF4evrwNpUBpP0MPeNzSxeV81Xpo+jb/e8qOOISBqK\nt5B8GigA/gj8KXz/qVSFkvSwv66ROxeUc+rQPlxVPCzqOCKSpuI9a2sXOkur0/nxn1ezfV8d910z\nhewsdbCLSOuOWkjMbB7gR5ru7jOSnkjSQsX2/Tz00lquKh7KacP7RR1HRNJYW3sk32uXFJJW3J07\n5pXRNTebr0wfF3UcEUlzRy0k4YWI0sk8W7aNv63ewewPTmBAzy5RxxGRNBdXH4mZvcW7D3HtAUqB\nb7n7zmQHk2gcrG/im/NXMHZQL645c0TUcUQkA8T7PJKngSbgN+Hw1YARFJNfAB9MejKJxE8XvcOm\n3Qf57fVnkpMd70l9ItKZxbulONvdb3H3t8LX14Dz3f0uoPBIC5nZdDNbZWYVZnZzK9NnmVmVmS0L\nX9eF4y+IGbfMzGrN7PJwWpGZvWpmq83sMTPTxQ1JsmFnDT9b9A4zJp3ImSP7Rx1HRDJEvIWkp5md\ncWjAzEqAnuFgY2sLhM91vxe4GJgAfNTMJrQy62PuPjl8PQjg7gsPjQOmATXAc+H8dwH3uPtoYBfw\nmTjbIG345oIV5GQZt14yPuooIpJB4i0k1wEPmtlaM1sHPAhcZ2Y9gO8cYZkSoMLd17h7PfBbYOZx\nZLwSeNrdayy4W+A04PFw2iPA5cexTmlh4artPL9iG5+fNpoT+nSNOo6IZJB4L0hcApxiZn0Inpa4\nO2by746w2BBgY8xwJXBGK/NdYWbnAW8D/+nuG1tMvxr4Qfi+P7Db3Q/tBVWGn/MuZnY9cD3A8OHD\njxBRAOoam5gzbwUjB/Tg0+cURh1HRDJMvLeR72NmPyB4XvsLZvb9sKgcdbFWxrU882seUOjupwIv\nEOxhxH7uYOAU4NljWGcw0v1+dy929+KCgoI2onZuP39pLWt3HGD2jIl0ydGDL0Xk2MR7aOshYB9w\nVfjaCzzcxjKVQOwNmoYCm2NncPed7l4XDj4ATGmxjqsInsrYEA7vAPqa2aE9qXetU47Nlj0H+cmf\nK7howiDOH6OCKyLHLt5CcpK7zw77O9a4+x3AyDaWWQKMDs+yyiM4RDU3doZwj+OQGUB5i3V8FHj0\n0IC7O8Ejfq8MR10LPBlnG6QV335qJU3Nzjcua+08CBGRtsVbSA6a2TmHBszsbODg0RYI+zFuIDgs\nVQ78Lnze+xwzO3SPrhvNrMzM3iC4KeSsmM8oJNijaXl1/VeBm8ysgqDP5OdxtkFaeOWdncx7YzOf\nPf8khuV3jzqOiGQoC37ktzGT2WSC/os+BP0U1cAsd38jtfGSo7i42EtLS6OOkVYampq57EcvcaC+\nkRduOp+uueobEZF/ZmZL3b24rfniPWtrGTDJzHqHw3sTzCcR+9Ur61m1bR/3XTNFRUREEtLWbeRv\nOsJ4ANz9B61Nl/RWta+Oe55/m/PGFHDRhEFRxxGRDNfWHkmvdkkh7eq7z6yktrGJ2R+ccPhHgYjI\n8WrrNvJ3tFcQaR+vbdjF75dW8m/nj+Skgp5tLyAi0oZjvr2rmb2WiiCSek3NzuwnyxjUuwufnzY6\n6jgi0kEcz33CdSwkQz22ZCNvbdrDrZeMp2eXeJ8gICJydMdTSBYkPYWk3O6aeu5+diUlRfnMmHRi\n1HFEpAM55kLi7l9PRRBJre8/9zZ7axu5Y8ZEdbCLSFLFe9PGfWa2t8Vro5n9yczaulWKRGz5pj38\n+tX1XHPmCMYP7h11HBHpYOI9UP4Dgpsj/oagj+Rq4ARgFcENHd+binCSOHdn9twy+nXP4z8vHBN1\nHBHpgOI9tDXd3e9z933uvtfd7wcucffHgH4pzCcJ+tPrm1i6fhdfnT6OPt1yo44jIh1QvIWk2cyu\nMrOs8HVVzLS2b9YlkdhX28C3n1rJpGF9uXLK0KjjiEgHFW8h+ThwDbAd2Ba+/4SZdSO4w6+koR+9\nuJqdB+qYM2MiWVnqYBeR1Ij3po1rgA8eYfJLyYsjybJ62z4efnkdV08dxqRhfaOOIyIdWLxnbY0x\nsxfNbHk4fKqZ6TTgNOXu3D6vjO552XzporFRxxGRDi7eQ1sPALcADQDu/ibBmVuShp5evpWXK3by\npQ+MpX/PLlHHEZEOLt5C0t3dF7cY15jsMJK4mvpGvjV/BeMH9+ZjJcOjjiMinUC8hWSHmZ1EeIaW\nmV0JbElZKjluP/3LO2zeU8ucmRPJyT6eO+CIiBybeC9I/A/gfmCcmW0C1hKcySVpZN2OA9y3aA0f\nOm0IUwvzo44jIp1EvIVkE/AwsBDIB/YC1wJzUpRLjsM3568gN9u45eJxUUcRkU4k3kLyJLAbeI3g\nVimSZl4s38aLK7dz6yXjGNi7a9RxRKQTibeQDHX36SlNIsettqGJOfNXcFJBD2adVRR1HBHpZOLt\njf0/MzslpUnkuD34tzWs31nD7TMmkpejDnYRaV/x7pGcA8wys7VAHcEdgN3dT01ZMonLpt0H+cnC\nCi4++QTOHV0QdRwR6YTiLSQXpzSFHLdvLygH4GuXjo84iYh0VvHea2t9qoPIsXu5YgcL3trCTReO\nYWi/7lHHEZFOSgfUM1RDUzOz55YxPL8715+nh1SKSHRUSDLUI/+3jort+7ntsgl0zc2OOo6IdGIq\nJBlo+75afvjCai4YW8D7xg+MOo6IdHIqJBnov59eSX1jM7d9cCJmemCViERLhSTDlK6r5o+vbeK6\nc4soGtAj6jgiIiokmaSp2bntyTIG9+nKDdNGRR1HRARQIckov1m8gRVb9vK1S8fTPS/eS4BERFIr\npYXEzKab2SozqzCzm1uZPsvMqsxsWfi6LmbacDN7zszKzWyFmRWG439hZmtjlpmcyjaki+oD9Xzv\n2VW8Z2R/Lj1lcNRxREQOS9nPWjPLBu4FLgQqgSVmNtfdV7SY9TF3v6GVVfwSuNPdnzeznkBzzLQv\nu/vjKQmepr733Cr21zVyx0x1sItIeknlHkkJUOHua9y9HvgtMDOeBc1sApDj7s8DuPt+d69JXdT0\n9lblHh5dvIFZZxUyZlCvqOOIiPyTVBaSIcDGmOHKcFxLV5jZm2b2uJkNC8eNAXab2R/N7HUzuzvc\nwznkznCZe8ysS4ryp4XmZue2ucvp36MLX3j/6KjjiIi8SyoLSWvHX7zF8DygMLyL8AvAI+H4HOBc\n4EvAVGAkMCucdgswLhyfD3y11Q83u97MSs2stKqqKoFmROsPr1Xy+obd3HzxOHp3zY06jojIu6Sy\nkFQCw2KGh9Li6YruvtPd68LBB4ApMcu+Hh4WawSeAE4Pl9nigTqCx/+WtPbh7n6/uxe7e3FBQWbe\nXn1vbQN3PbOS04f35cOntbYzJyISvVQWkiXAaDMrMrM84GpgbuwMZhZ7+tEMoDxm2X5mdqgCTANW\nxC5jQY/z5cDylLUgYj98fjU7D9QzZ+bJZGWpg11E0lPKztpy90YzuwF4FsgGHnL3MjObA5S6+1zg\nRjObATQC1YSHr9y9ycy+BLwYFoylBHssAL8OC4wBy4DPpqoNUVq1dR+PvLKOj5UM5+QhfaKOIyJy\nRObestui4ykuLvbS0tKoY8TN3fnoA39n5dZ9LPyv99KvR17UkUSkEzKzpe5e3NZ8urI9Dc1/cwt/\nX1PNly4aqyIiImlPhSTNHKhr5M4F5Uw8sTcfLRkedRwRkTbphk1p5t6FFWzdW8u9Hz+NbHWwi0gG\n0B5JGllTtZ8H/raGK04fypQR+VHHERGJiwpJmnB37pi3gq452Xz14rFRxxERiZsKSZp4oXw7i96u\n4gvvH83AXl2jjiMiEjcVkjRQ29DEnPlljB7Yk2vPKow6jojIMVFnexq4b9EaNlYf5DfXnUFutmq7\niGQWbbUitrG6hv/9SwWXnjqYs0YNiDqOiMgxUyGJ2J0Lysky42uXjI86iojIcVEhidBf367imbKt\n3DBtFCf27RZ1HBGR46JCEpH6xmZun1dGYf/uXHduUdRxRESOmzrbI/Lwy2tZU3WAh2dNpUtOdtsL\niIikKe2RRGDb3lp+9OJq3j9+IBeMGxh1HBGRhKiQROA7T5XT0Ox847IJUUcREUmYCkk7e3XNTp5Y\ntpnPnjeSEf17RB1HRCRhKiTtqLGpmdlzyxjStxufe++oqOOIiCSFCkk7+vWrG1i5dR9fv3Q83fLU\nwS4iHYMKSTvZub+O7z+3inNGDWD6ySdEHUdEJGlUSNrJ3c+uoqa+idtnTMBMD6wSkY5DhaQdLNu4\nm8dKN/Lpc4oYNbBX1HFERJJKhSTFmpud2U8uZ0DPLnx+mjrYRaTjUSFJsd8v3cgblXu49ZJx9Oqa\nG3UcEZGkUyFJoT01Ddz1zCqmFvbj8slDoo4jIpISKiQpdM8Lb7O7pp7bZ0xUB7uIdFgqJClSvmUv\nv3xlHZ84cwQTT+wTdRwRkZRRIUkBd2f2k2X07Z7HTReOiTqOiEhKqZCkwNw3NrN4XTVf/sBY+nbP\nizqOiEhKqZAk2f66Ru5cUM6pQ/twVfGwqOOIiKScHmyVZD/+82q276vjvmumkJ2lDnYR6fi0R5JE\nFdv389BLa7mqeCinDe8XdRwRkXahQpIk7s4d88rompvNV6aPizqOiEi7USFJkmfLtvG31Tu46cIx\nDOjZJeo4IiLtJqWFxMymm9kqM6sws5tbmT7LzKrMbFn4ui5m2nAze87Mys1shZkVhuOLzOxVM1tt\nZo+ZWeSnRR2sb+Kb81cwdlAvrjlzRNRxRETaVcoKiZllA/cCFwMTgI+aWWsPKX/M3SeHrwdjxv8S\nuNvdxwMlwPZw/F3APe4+GtgFfCZVbYjXTxe9w6bdB7lj5kRysrWTJyKdSyq3eiVAhbuvcfd64LfA\nzHgWDAtOjrs/D+Du+929xoL7jEwDHg9nfQS4PPnR47dhZw0/W/QOMyadyJkj+0cZRUQkEqksJEOA\njTHDleG4lq4wszfN7HEzO3ThxRhgt5n90cxeN7O7wz2c/sBud29sY53t5psLVpCTZdx6yfgoY4iI\nRCaVhaS1iyi8xfA8oNDdTwVeINjDgOD6lnOBLwFTgZHArDjXGXy42fVmVmpmpVVVVceePg4LV23n\n+RXbuPF9ozmhT9eUfIaISLpLZSGpBGIv7R4KbI6dwd13untdOPgAMCVm2dfDw2KNwBPA6cAOoK+Z\n5RxpnTHrvt/di929uKCgICkNilXX2MSceSsYOaAHnz67KOnrFxHJFKksJEuA0eFZVnnA1cDc2BnM\nbHDM4AygPGbZfmZ2qAJMA1a4uwMLgSvD8dcCT6Yo/1H9/KW1rN1xgNkzJpKXow52Eem8UrYFDPck\nbgCeJSgQv3P3MjObY2YzwtluNLMyM3sDuJHg8BXu3kRwWOtFM3uL4JDWA+EyXwVuMrMKgj6Tn6eq\nDUeyZc9BfvLnCi6aMIjzxyR/b0dEJJNY8CO/YysuLvbS0tKkre/zj77Oc2VbeeGm8xmW3z1p6xUR\nSSdmttTdi9uaT8dkjtEr7+xk3hub+dx7T1IRERFBheSYNDQ1c/vcMob268Znzz8p6jgiImlBheQY\n/OqV9azato9vXDaBrrnZUce/JzAZAAAG6klEQVQREUkLKiRxqtpXxz3Pv815Ywq4aMKgqOOIiKQN\nFZI4ffeZldQ2NjH7gxMI7tQiIiKgQhKX1zbs4vdLK/nMOSM5qaBn1HFERNKKCkkbmpqd2U+WMah3\nFz4/bVTUcURE0o4KSRseW7KRtzbt4dZLxtOjix5xLyLSkgrJUeyuqefuZ1dSUpTPjEknRh1HRCQt\nqZAcxfefe5u9tY3cMWOiOthFRI5AheQohvbrxvXnjWT84N5RRxERSVs66H8U/6ar10VE2qQ9EhER\nSYgKiYiIJESFREREEqJCIiIiCVEhERGRhKiQiIhIQlRIREQkISokIiKSEHP3qDOknJlVAeuPc/EB\nwI4kxolSR2lLR2kHqC3pqqO0JdF2jHD3grZm6hSFJBFmVuruxVHnSIaO0paO0g5QW9JVR2lLe7VD\nh7ZERCQhKiQiIpIQFZK23R91gCTqKG3pKO0AtSVddZS2tEs71EciIiIJ0R6JiIgkRIUkZGbTzWyV\nmVWY2c2tTO9iZo+F0181s8L2T9m2ONoxy8yqzGxZ+LouipzxMLOHzGy7mS0/wnQzsx+FbX3TzE5v\n74zxiKMd7zWzPTHfyW3tnTFeZjbMzBaaWbmZlZnZF1qZJ+2/lzjbkRHfi5l1NbPFZvZG2JY7Wpkn\ntdsvd+/0LyAbeAcYCeQBbwATWszz78DPwvdXA49Fnfs42zEL+EnUWeNsz3nA6cDyI0y/BHgaMOBM\n4NWoMx9nO94LzI86Z5xtGQycHr7vBbzdyr+xtP9e4mxHRnwv4d9zz/B9LvAqcGaLeVK6/dIeSaAE\nqHD3Ne5eD/wWmNlinpnAI+H7x4H3Wfo9yD2edmQMd/8rUH2UWWYCv/TA34G+Zja4fdLFL452ZAx3\n3+Lur4Xv9wHlwJAWs6X99xJnOzJC+Pe8PxzMDV8tO79Tuv1SIQkMATbGDFfy7n9Uh+dx90ZgD9C/\nXdLFL552AFwRHnJ43MyGtU+0lIi3vZngPeGhiafNbGLUYeIRHh45jeAXcKyM+l6O0g7IkO/FzLLN\nbBmwHXje3Y/4naRi+6VCEmitMres6PHME7V4Ms4DCt39VOAF/vErJRNlwncSj9cIbkUxCfgx8ETE\nedpkZj2BPwBfdPe9LSe3skhafi9ttCNjvhd3b3L3ycBQoMTMTm4xS0q/ExWSQCUQ+8t8KLD5SPOY\nWQ7Qh/Q7XNFmO9x9p7vXhYMPAFPaKVsqxPO9pT1333vo0IS7PwXkmtmAiGMdkZnlEmx8f+3uf2xl\nloz4XtpqR6Z9LwDuvhv4CzC9xaSUbr9USAJLgNFmVmRmeQSdUXNbzDMXuDZ8fyXwZw97rtJIm+1o\ncax6BsGx4Uw1F/hkeJbQmcAed98SdahjZWYnHDpebWYlBP8vd0abqnVhzp8D5e7+gyPMlvbfSzzt\nyJTvxcwKzKxv+L4b8H5gZYvZUrr9yknWijKZuzea2Q3AswRnPj3k7mVmNgcodfe5BP/ofmVmFQSV\n/OroErcuznbcaGYzgEaCdsyKLHAbzOxRgjNnBphZJTCboCMRd/8Z8BTBGUIVQA3wqWiSHl0c7bgS\n+JyZNQIHgavT8EfKIWcD1wBvhcfkAW4FhkNGfS/xtCNTvpfBwCNmlk1Q7H7n7vPbc/ulK9tFRCQh\nOrQlIiIJUSEREZGEqJCIiEhCVEhERCQhKiQiIpIQFRKRJDCz/W3PddTlHzezkeH7nmZ2n5m9E97N\n9a9mdoaZ5YXvddq+pBUVEpGIhfdwynb3NeGoBwnO9R/t7hMJrvUZEN6I80XgI5EEFTkCFRKRJAqv\n5r7bzJab2Vtm9pFwfJaZ/W+4hzHfzJ4ysyvDxT4OPBnOdxJwBvB1d28GCO/mvCCc94lwfpG0oV1k\nkeT6MDAZmAQMAJaY2V8JrqQuBE4BBhLcmuahcJmzgUfD9xOBZe7edIT1LwempiS5yHHSHolIcp0D\nPBrejXUbsIhgw38O8Ht3b3b3rcDCmGUGA1XxrDwsMPVm1ivJuUWOmwqJSHId6WFBR3uI0EGga/i+\nDJhkZkf7v9kFqD2ObCIpoUIiklx/BT4SPmiogOAxu4uBlwgeKJZlZoMIbuJ4SDkwCsDd3wFKgTti\n7jw72sxmhu/7A1Xu3tBeDRJpiwqJSHL9CXgTeAP4M/CV8FDWHwieCbEcuI/gaXx7wmUW8M+F5Trg\nBKDCzN4ieG7Moed5XEBwd12RtKG7/4q0EzPr6e77w72KxcDZ7r41fIbEwnD4SJ3sh9bxR+AWd1/V\nDpFF4qKztkTaz/zwAUR5wDfDPRXc/aCZzSZ4rvaGIy0cPqzsCRURSTfaIxERkYSoj0RERBKiQiIi\nIglRIRERkYSokIiISEJUSEREJCEqJCIikpD/B4OUoSvWCqmhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xfd67710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "\n",
    "n_classes = 5\n",
    "\n",
    "array = lr_cv_L2.scores_\n",
    "\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "   # print(np.mean(tuple(array.values())[0][j],axis = 0))\n",
    "    print tuple(array.values())[0][j]\n",
    "    scores[j][:] = tuple(array.values())[0][j]\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='lbfgs', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', multi_class='ovr')\n",
    "lrcv_L2.fit(train_X, train_y)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.63182653, -0.65442458, -0.6576393 , -0.65795593],\n",
       "        [-0.61746408, -0.64772533, -0.65158617, -0.65199871],\n",
       "        [-0.49078747, -0.4966887 , -0.49761857, -0.49771689],\n",
       "        [-0.59120523, -0.61283346, -0.61572387, -0.61602504],\n",
       "        [-0.47277912, -0.48290569, -0.48430176, -0.48444911]])}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.63182653 -0.65442458 -0.6576393  -0.65795593]\n",
      "[-0.61746408 -0.64772533 -0.65158617 -0.65199871]\n",
      "[-0.49078747 -0.4966887  -0.49761857 -0.49771689]\n",
      "[-0.59120523 -0.61283346 -0.61572387 -0.61602504]\n",
      "[-0.47277912 -0.48290569 -0.48430176 -0.48444911]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xt4VdWd//H3l4SEOyQQLnJJQEAu\niiARaana0tqiFLSjT7WtbbH1Z+vUsTOtTodeFLV1bK21tnXaqmPVPh1t62gFRAtWq9OLShBEwzVy\nDdcAIQEDuX5/f+wNHNMAhyQ7+5zk83qePJ6999qb7/LA+matvfda5u6IiIg0V6e4AxARkfSmRCIi\nIi2iRCIiIi2iRCIiIi2iRCIiIi2iRCIiIi2iRCIiIi2iRCIiIi2iRCIiIi2SGXcAbaFfv35eUFAQ\ndxgiImll2bJle9w972TlOkQiKSgooKioKO4wRETSipltTqachrZERKRFlEhERKRFlEhERKRFlEhE\nRKRFlEhERKRFlEhERKRFlEhERKRFOsR7JCIiqa6hwampb6C2voGaugZq6z34HO6rrTt2/FiZBmrq\nndq6hP1Hzgv3XTNtOLndsyKNXYlERNq1xAa6tlEjm9hY1yQ2xnV+9HNiY11T33C00T7SYCee897j\nTf+ZNWFSaFymvsFbve5mMPvs05RIRCT1VdfVU1FVGzaMxxrOYw1r44YzaEyrExre9zTOCY11YgN/\nvN/Aa+rqj/4ZjX+jr4uggQbIyuxEVkYnOmcYnTM60TmjE1mZwXbw3+Cna+cMenXJDLYTzjlSJivj\nWNnOmfae7aPXO3q80XZYJis89+i+8M/I6GSYWST1T6REIiKnbO/Baoo2l/PG5nKKNpfzVmkFNfUN\nrXLto41z5rFG9kiDmthQZnfuRI8umWEjevxG/T2N7dGf9zb2RxrnxH1ZCQ3z0cY5bOjbqoFOF0ok\nInJC7s47ZQcp2hQkjWWby9m4510gaPTPGtKba6YVMDS329EG+1iDbEcb+vdsJ5RJ/G06Uw10WlIi\nEZH3OFxbz8rSCoo272PZpnKWbSlnf1UtALndszhnWA5XnjuUwvwczhzcmy6dM2KOWOKmRCLSwZUd\nqGbZ5n1HexzF2yuorQ/uK5ye152PjRvI5IIcCvNzGN6vu3oM8g+USEQ6kIYGp+ToMNU+lm0uZ/Pe\nKiC4eXz2kN588QMjKMzP4Zz8nMif9pH2QYlEpB07VFPPiq37WRYmjWWby6k8XAdAvx5ZTM7P4TPn\nDWNyfi5nDu5FdqaGqeTUKZGItCO7Kw9TtLmcok3lLNu8j+LtlUcffx3VvwczJwxicn4uhfk55Pft\npmEqaRVKJCJpqr7BWbfrwNGeRtHmfWzddwiA7MxOnD20D9ddMILCghzOGZZDn24appJoKJGIpImq\nmjpWbNkf9Dg2l7N8czkHqoNhqrye2RTm5/D59xUwOT+H8af1JitTU+lJ21AiEUlROysOUxQ+TbVs\nczmrdlRS3+CYwej+PZk18TQK83MozM9laG5XDVNJbCJNJGY2A7gPyAAecve7Gh2fA9wNbAt3/czd\nHwqP/QCYSTBD8RLgq+7uZjYZeAToCiw6sj/KeohErb7BWbOzMhiiChPHtv3BMFWXzp2YOLQP1194\nOpPDYareXTvHHLHIMZElEjPLAO4HLgJKgaVmNt/dVzUq+lt3v6HRue8HpgETwl1/AS4E/gz8HLgO\neJUgkcwAnouoGiKROFh9ZJgqeJpq+Zb9HAyHqQb0yqYwP5cvfmA4hQU5jB3Ui84ZGqaS1BVlj2QK\nUOLuGwDM7AngUqBxImmKA12ALMCAzsAuMxsE9HL3v4fXfAy4DCUSSXHb9h+iaNO+oz2ONTsrafBg\ndtYzBvTkskmnUZify+T8HIbkaJhK0kuUiWQwsDVhuxQ4r4lyl5vZBcA64N/cfau7/93MXgJ2ECSS\nn7n7ajMrDK+TeM3B0YQv0jx19Q2s2XmAok37js5NtaPiMADdsjKYNKwPN0wfxeT8HCYN60OvLhqm\nkvQWZSJp6leqxvcyFgCPu3u1mX0ZeBSYbmYjgbHAkLDckjDZHErimsEfbnYdwRAYw4YNa0b4Ismp\nPFzL8i37WbZpH8u2BMNUVTX1AAzq3YXJ+cH0IoUFuYwZ2JNMDVNJOxNlIikFhiZsDwG2JxZw970J\nmw8C3w8/fwJ41d0PApjZc8BU4NccSy5NXjPh2g8ADwAUFhbqZry0CnentPzQ0fc2ijaVs3bXAdyh\nk8HYQb24YvKQIHkU5DK4T9e4QxaJXJSJZCkwysyGEzyVdRXw6cQCZjbI3XeEm7OB1eHnLcD/M7P/\nJOjZXAj82N13mNkBM5sKvAZ8DvhphHWQDq62voHVOyqPPklVtHkfuyqrAeiRncmkYX2YceZACvNz\nmTisDz2y9US9dDyR/a139zozuwH4I8Hjvw+7e7GZ3Q4Uuft84EYzmw3UAfuAOeHpTwLTgbcIhq6e\nd/cF4bHrOfb473PoRru0oopDtbyxpZxl4aSGb26t4FBtMEw1uE9Xzhvel8KCHCbn5zBmYC8yOumm\nuIh1hFcwCgsLvaioKO4wJMW4O1v3HQqGqDYHyWPd7mCYKqOTMW5QLybn54TDVDkM6q1hKulYzGyZ\nuxeerJz64dJh1NQ1ULy94thLf1vKKTsQDFP1zM5kUn4OMycMojA/h7OH9qG7hqlEkqJ/KdJu7a+q\n4Y0t5UcXbHpz636q64J1xYfkdGXa6X2ZXBDMhDt6QE8NU4k0kxKJtAvuzqa9VeFMuMHTVOt3HwQg\ns5Mx/rRefOa8/KP3Nwb06hJzxCLthxKJpKXqunre3lZ5NGm8saWcPQdrAOjVJZNz8nO4dOJpTM7P\n5eyhvemWpb/qIlHRvy5JKzsrDvP1369g6aZyasJhqmG53bhgVF64rnguo/r3oJOGqUTajBKJpJXb\nFxZTtKmcz00NhqnOyc+hf08NU4nESYlE0sZfS/aw6K2d/NtHRvPVj4yKOxwRCWnSH0kLtfUNzJtf\nzJCcrnzpwhFxhyMiCZRIJC089vfNrN99kO98fBxdOmfEHY6IJFAikZRXdqCaHy9Zx/mj+vHRcQPi\nDkdEGlEikZT3g+fXcLiunnmzx2vBJ5EUpEQiKW35lnJ+v6yUL0wbzul5PeIOR0SaoEQiKauhwZk3\nv5j+PbP5lw/rKS2RVKVEIinr98u28mZpBXMvGaN1PkRSmBKJpKSKQ7X84Pm1FObncNnEwXGHIyIn\noF/zJCXdu2Qd+6pqeHT2FN1gF0lx6pFIylmzs5Jfv7qZT08ZxpmDe8cdjoichBKJpBR359ZniunZ\nJZObPnpG3OGISBKUSCSlLFy5g9c27uPrHz2DnO5ZcYcjIklQIpGUUVVTx52LVjNuUC8+PWVY3OGI\nSJJ0s11Sxv0vlbCj4jA/+dQkLXsrkkbUI5GUsGnPuzz4ykYum3ga5xbkxh2OiJwCJRJJCXcsXEXn\nDGPuJWPjDkVETpESicTupTW7+dOa3dz44VEM6KXVDkXSjRKJxKq6rp7bFhQzIq8710wbHnc4ItIM\nSiQSq//+y0Y27a3i1lnjycrUX0eRdKR/uRKbnRWH+dmLJVw0bgAXjs6LOxwRaSYlEonNnYtWU9fg\nfGfmuLhDEZEWUCKRWLy6YS/z39zOly8YwbC+3eIOR0RaQIlE2lxdfQPz5hczuE9Xrv/gyLjDEZEW\nUiKRNveb17awZucBvjVzLF2zMuIOR0RaSIlE2tTeg9Xcs3gt7z+9LxefOTDucESkFSiRSJv64eK1\nvFtTz7zZ47VglUg7oUQibWZl6X6eWLqVz7+vgNEDesYdjoi0EiUSaRMNDc6t84vp2z2Lf71oVNzh\niEgrUiKRNvHU8m0s37Kfb8wYQ68uneMOR0RakRKJRK7ycC13PbeGScP6cPk5Q+IOR0RamRa2ksj9\n5IX17H23mofnFNJJC1aJtDvqkUikSnYf4JG/beLKwqFMGNIn7nBEJAKRJhIzm2Fma82sxMz+o4nj\nc8yszMxWhD/Xhvs/lLBvhZkdNrPLwmOPmNnGhGMTo6yDNJ+7M2/+KrplZXDzx86IOxwRiUhkQ1tm\nlgHcD1wElAJLzWy+u69qVPS37n5D4g53fwmYGF4nFygBFicUudndn4wqdmkdz7+9k7+U7GHerHH0\n7ZEddzgiEpEoeyRTgBJ33+DuNcATwKXNuM4VwHPuXtWq0UmkDtXU891nVzNmYE+unpofdzgiEqEo\nE8lgYGvCdmm4r7HLzWylmT1pZkObOH4V8Hijfd8Lz7nXzJr8VdfMrjOzIjMrKisra1YFpPl+/vI7\nbNt/iHmzx5OZoVtxIu1ZlP/Cm3o8xxttLwAK3H0C8ALw6HsuYDYIOAv4Y8LuucAY4FwgF/hGU3+4\nuz/g7oXuXpiXp0WT2tLWfVX84uV3+PiEQUwd0TfucEQkYlEmklIgsYcxBNieWMDd97p7dbj5IDC5\n0TU+CTzt7rUJ5+zwQDXwK4IhNEkhdyxcRYYZ35o5Nu5QRKQNRJlIlgKjzGy4mWURDFHNTywQ9jiO\nmA2sbnSNT9FoWOvIORbM+HcZ8HYrxy0t8Mq6Mhav2sUN00cyqHfXuMMRkTYQ2VNb7l5nZjcQDEtl\nAA+7e7GZ3Q4Uuft84EYzmw3UAfuAOUfON7MCgh7Ny40u/RszyyMYOlsBfDmqOsipqalrYN6CYgr6\nduPa84fHHY6ItJFI32x390XAokb7bkn4PJfgnkdT526iiZvz7j69daOU1vLI3zayoexdHp5TSHam\nFqwS6Sj0OI20it2Vh7nvhfVMH9Of6WMGxB2OiLQhJRJpFXc9t4baeueWj4+LOxQRaWNKJNJiRZv2\n8dTybVx7/nAK+nWPOxwRaWNKJNIi9Q3OLc8UM7BXF77yoZFxhyMiMUgqkZjZNDPrHn6+2sx+ZGaa\n90J4/PUtrNpRyTdnjqV7tlYlEOmIku2R/ByoMrOzgX8HNgOPRRaVpIXyd2v44eK1nDc8l1kTBp38\nBBFpl5JNJHXu7gSTLt7n7vcBPaMLS9LBPUvWUnmolnmzxxO8HyoiHVGyYxEHzGwucDVwQThFvBbe\n7sCKt1fwP69t4bNT8xk7qFfc4YhIjJLtkVwJVANfdPedBC8K3h1ZVJLSggWriunTLYuvXaQFq0Q6\nuqR7JARDWvVmNppg9t3GU7tLB/HMiu0s3VTOXf90Fr27qWMq0tEl2yN5Bcg2s8HAn4BrgEeiCkpS\n18HqOu5ctJoJQ3rzycKmlo8RkY4m2URi4QqF/wT81N0/AYyPLixJVT99cT27D1Rz2+zxdOqkG+wi\ncgqJxMzeB3wGeDbcp1n5Oph3yg7y8F82csXkIUwalhN3OCKSIpJNJP9KMEvv0+FU8COAl6ILS1KN\nu3P7glV0yczgGzPGxB2OiKSQpG62u/vLwMtm1tPMerj7BuDGaEOTVPLC6t28vK6Mb88cS17P7LjD\nEZEUkuwUKWeZ2XKC1QhXmdkyM9M9kg7icG09ty8sZlT/Hnz+/QVxhyMiKSbZoa1fAl9z93x3HwZ8\nnWCNdekAHnhlA1v3HWLe7PF0ztA8nyLyXsm2Ct3d/eg9EXf/M6D5wjuA0vIq/uvPJVx85kCmjewX\ndzgikoKSfSFxg5l9B/h1uH01sDGakCSV3LloNQDfmjk25khEJFUl2yP5ApAHPAU8HX6+JqqgJDX8\ntWQPi97ayfUXjmRITre4wxGRFJXsU1vl6CmtDqW2voF584sZktOVL104Iu5wRCSFnTCRmNkCwI93\n3N1nt3pEkhIe+/tm1u8+yAOfnUyXznr3VESO72Q9kh+2SRSSUsoOVPPjJeu4YHQeF40bEHc4IpLi\nTphIwhcRpYP5wfNrOFxXz62zxmnBKhE5qaTukZjZW/zjEFcFUAR81933tnZgEo/lW8r5/bJSvnTB\nCE7P6xF3OCKSBpJ9/Pc5oB74n3D7KsAIkskjwKxWj0zaXENDsGBV/57Z/MuHR8UdjoikiWQTyTR3\nn5aw/ZaZ/dXdp5nZ1VEEJm3v98u28mZpBfdeeTY9spP9qyEiHV2y75H0MLPzjmyY2RTgyLhHXatH\nJW2uoqqW7z+/lsL8HC6bODjucEQkjST7a+e1wMNm1oNgSKsS+KKZdQf+M6rgpO3c+8I6yqtqmDd7\nim6wi8gpSfaFxKXAWWbWm2C1xP0Jh38XSWTSZtbsrOTXr27m01OGcebg3nGHIyJpJtlp5Hub2Y8I\n1mt/wczuCZOKpDl359ZniunZJZObPnpG3OGISBpK9h7Jw8AB4JPhTyXwq6iCkrazcOUOXtu4j69/\n9AxyumfFHY6IpKFk75Gc7u6XJ2zfZmYroghI2k5VTR13LlrNuEG9+PSUYXGHIyJpKtkeySEz+8CR\nDTObBhyKJiRpK/e/VMKOisPcful4MjrpBruINE+yPZLrgUeP3GwH9gFzogpKordpz7s8+MpGPjFp\nMIUFuXGHIyJpLNmntlYAZ5tZr3C7MtKoJHJ3LFxF5wxj7sVj4g5FRNLcyaaR/9px9gPg7j+KICaJ\n2EtrdvOnNbuZe/EY+vfqEnc4IpLmTtYj6dkmUUibqa6r57YFxYzI684104bHHY6ItAMnm0b+tpZc\n3MxmAPcBGcBD7n5Xo+NzgLuBbeGun7n7Q2b2IeDehKJjgKvc/Q9mNhx4AsgF3gA+6+41LYmzI3no\n/zayaW8Vj35hClmZyT5rISJyfKfckpjZG0mWywDuBy4GxgGfMrNxTRT9rbtPDH8eAnD3l47sA6YD\nVcDisPz3gXvdfRRQDnzxVOvQUe2oOMTPXizhonEDuHB0XtzhiEg70ZxfSZN9TnQKUOLuG8IewxPA\npc34864AnnP3KgtuzkwHngyPPQpc1oxrdkh3LlpDvTvfmdlUPhcRaZ7mJJJnkyw3GNiasF0a7mvs\ncjNbaWZPmtnQJo5fBTwefu4L7Hf3IzMOH++a0sirG/ay4M3tfPmCEQzr2y3ucESkHTnlROLu306y\naFM9l8arLC4ACtx9AvACQQ/j2AXMBgFnAX88hWseOfc6Mysys6KysrIkQ26f6uobmDe/mMF9unL9\nB0fGHY6ItDPJTtp4wMwqG/1sNbOnzWzEcU4rBRJ7GEOA7YkF3H2vu1eHmw8Ckxtd45PA0+5eG27v\nAfqY2ZGHBP7hmgnXfsDdC929MC+vY98P+M1rW1iz8wDfmjmWrlkZcYcjIu1Msj2SHwE3EwwjDQFu\nImj4nyCY0LEpS4FRZjbczLIIhqjmJxYIexxHzAZWN7rGpzg2rIW7O/ASwX0TgM8DzyRZhw5p78Fq\n7lm8lmkj+3LxmQPjDkdE2qFkE8kMd/+lux9w90p3fwC4xN1/C+Q0dUJ4H+MGgmGp1cDv3L3YzG43\ns9lhsRvNrNjM3gRuJGHaFTMrIOjRvNzo0t8AvmZmJQT3TP47yTp0SD9cvJaqmnrmzRqvBatEJBLJ\nzrXVYGaf5NjTUlckHGvyHgWAuy8CFjXad0vC57nA3OOcu4kmbqS7+waCJ8LkJFaW7ueJpVv5wrTh\njBqgd0tFJBrJ9kg+A3wW2A3sCj9fbWZdCXodkmIaGpxb5xfTt3s2X/3IqLjDEZF2LNlJGzcAs45z\n+C+tF460lqeWb2P5lv3cfcUEenXpHHc4ItKOJfvU1mgz+5OZvR1uTzCzZB8DljZWebiWu55bw6Rh\nfbj8nCFxhyMi7VyyQ1sPEtzLqAVw95UET2FJCrrvhfXsfbea22aPp5MWrBKRiCWbSLq5++uN9tU1\nWVJitX7XAR792yauLBzKhCF94g5HRDqAZBPJHjM7nfAJLTO7AtgRWVTSLO7OvAXFdMvK4OaPnRF3\nOCLSQST7+O9XgAeAMWa2DdhI8CSXpJDn397JX0v2Mm/WOPr2yI47HBHpIJJNJNuAXxG8VZ4LVBK8\nVX57RHHJKTpUU893n13NmIE9uXpqftzhiEgHkmwieQbYT7CQVJNzW0m8fv7yO2zbf4gnrptKZoYW\nrBKRtpNsIhni7jMijUSabeu+Kn7x8jvMOvs0po7oG3c4ItLBJPur69/M7KxII5Fmu2PhKjLM+OYl\nY+IORUQ6oGR7JB8A5pjZRqCaYF0QD9cRkRi9sq6Mxat2cfPHzmBQ765xhyMiHVCyieTiSKOQZqmp\na2DegmIK+nbj2vOHxx2OiHRQyc61tTnqQOTUPfK3jWwoe5eH5xSSnakFq0QkHnq8J03tqjzMfS+s\nZ/qY/kwfMyDucESkA1MiSVN3PbeG2nrnlo+PizsUEenglEjSUNGmfTy9fBvXnj+cgn7d4w5HRDo4\nJZI0U9/g3PJMMQN7deErHxoZdzgiIkok6ebx17ewakcl35w5lu7ZyT50JyISHSWSNFL+bg0/XLyW\n84bnMmvCoLjDEREBlEjSyj1L1lJ5qJZ5s8djpgWrRCQ1KJGkieLtFfzPa1v43PsKGDuoV9zhiIgc\npUSSBtydefOL6dMti3/7yOi4wxEReQ8lkjTwzIrtLN1Uzr9/7Ax6d+scdzgiIu+hRJLiDlbXceei\n1UwY0ptPFg6NOxwRkX+g50dT3E9fXM/uA9X88rOT6dRJN9hFJPWoR5LC3ik7yMN/2cgVk4cwaVhO\n3OGIiDRJiSRFuTu3LVhFl8wMvjFDC1aJSOpSIklRS1bt4pV1ZXz1I6PI65kddzgiIselRJKCDtfW\nc8ezqxjVvweff39B3OGIiJyQbranoAde2cDWfYf4zbXn0TlDuV5EUptaqRRTWl7Ff/25hIvPHMi0\nkf3iDkdE5KSUSFLMnYtWA/CtmWNjjkREJDlKJCnkryV7WPTWTv75gyMZktMt7nBERJKiRJIiausb\nmDe/mKG5XbnughFxhyMikjQlkhTx2N83s373Qb4zcxxdOmfEHY6ISNKUSFJA2YFqfrxkHReMzuOi\ncQPiDkdE5JQokaSAHzy/hsN19dw6a5wWrBKRtBNpIjGzGWa21sxKzOw/mjg+x8zKzGxF+HNtwrFh\nZrbYzFab2SozKwj3P2JmGxPOmRhlHaL2xpZyfr+slC9MG87peT3iDkdE5JRF9kKimWUA9wMXAaXA\nUjOb7+6rGhX9rbvf0MQlHgO+5+5LzKwH0JBw7GZ3fzKSwNtQQ0OwYFX/ntn8y4dHxR2OiEizRNkj\nmQKUuPsGd68BngAuTeZEMxsHZLr7EgB3P+juVdGFGo/fFW1lZWkFcy8ZQ49sTTIgIukpykQyGNia\nsF0a7mvscjNbaWZPmtmRlZtGA/vN7CkzW25md4c9nCO+F55zr5ml5YyGFVW1/OCPaynMz+GyiU39\nbxERSQ9RJpKm7hp7o+0FQIG7TwBeAB4N92cC5wM3AecCI4A54bG5wJhwfy7wjSb/cLPrzKzIzIrK\nyspaUI1o3PvCOsqrapg3e7xusItIWosykZQCiWvDDgG2JxZw973uXh1uPghMTjh3eTgsVgf8ATgn\nPGeHB6qBXxEMof0Dd3/A3QvdvTAvL6/VKtUa1uys5NevbubTU4Zx5uDecYcjItIiUSaSpcAoMxtu\nZlnAVcD8xAJmNihhczawOuHcHDM7kgGmA6sSz7Hg1/jLgLcjq0EE3J1bnymmZ5dMbvroGXGHIyLS\nYpHd4XX3OjO7AfgjkAE87O7FZnY7UOTu84EbzWw2UAfsIxy+cvd6M7sJ+FOYMJYR9FgAfhMmGANW\nAF+Oqg5RWLhyB69t3Md3LzuTnO5ZcYcjItJi5t74tkX7U1hY6EVFRXGHQVVNHR++52Vyu2cx/4YP\nkNFJ90ZEJHWZ2TJ3LzxZOT1z2obuf6mEHRWH+emnJimJiEi7oSlS2simPe/y4Csb+cSkwRQW5MYd\njohIq1EiaSO3L1xF5wxj7sVj4g5FRKRVKZG0gRfX7OLFNbu58cOj6N+rS9zhiIi0KiWSiFXX1XP7\nglWMyOvONdOGxx2OiEirUyKJ2EP/t5FNe6u4ddZ4sjL1v1tE2h+1bBHaUXGIn71YwkXjBnDh6NR6\nu15EpLUokUTozkVrqHfnOzPHxR2KiEhklEgi8uqGvSx4cztfvmAEw/p2izscEZHIKJFEoK6+gXnz\nixncpyvXf3Bk3OGIiERKiSQCv3ltC2t2HuDbM8fSNSvj5CeIiKQxJZJWtvdgNfcsXsu0kX2ZcebA\nuMMREYmcEkkr++HitVTV1DNvlhasEpGOQYmkFa0s3c8TS7fy+fcXMGpAz7jDERFpE0okraShwbnl\nmWL6ds/mqx8ZFXc4IiJtRomklfzvG6Ws2Lqfb8w4g15dOscdjohIm1EiaQWVh2v5/vNrmDSsD5ef\nMyTucERE2pQWtmoF972wnr3v1vDwnHPppAWrRKSDUY+khdbvOsCjf9vElYVDmTCkT9zhiIi0OSWS\nFnB35i0opltWBjd/7Iy4wxERiYUSSQs8//ZO/lqyl69dNJq+PbLjDkdEJBZKJM10qKae7z67mjED\ne3L11Py4wxERiY1utjfTz19+h237D/HEdVPJzFA+FpGOSy1gM2zdV8UvXn6HWWefxtQRfeMOR0Qk\nVkokzXDHwlVkmPHNS8bEHYqISOyUSE7Ry+vKWLxqFzdMH8mg3l3jDkdEJHZKJKegpq6B2+YXU9C3\nG9eePzzucEREUoISySn41V83smHPu9wyaxzZmVqwSkQElEiStqvyMD/503qmj+nP9DED4g5HRCRl\nKJEk6a7n1lBb79zy8XFxhyIiklKUSJJQtGkfTy/fxrXnD6egX/e4wxERSSlKJCdRHy5YNbBXF77y\noZFxhyMiknKUSE7i8de3sGpHJd+aOZbu2ZoIQESkMSWSEyh/t4YfLl7L1BG5fHzCoLjDERFJSUok\nJ3DPkrUcOFzHvNnjMdOCVSIiTVEiOYGhOd247oIRjBnYK+5QRERSlgb9T+BLF54edwgiIilPPRIR\nEWkRJRIREWmRSBOJmc0ws7VmVmJm/9HE8TlmVmZmK8KfaxOODTOzxWa22sxWmVlBuH+4mb1mZuvN\n7LdmlhVlHURE5MQiSyRmlgHcD1wMjAM+ZWZNzS/yW3efGP48lLD/MeBudx8LTAF2h/u/D9zr7qOA\ncuCLUdVBREROLsoeyRSgxN03uHsN8ARwaTInhgkn092XALj7QXevsuAZ3OnAk2HRR4HLWj90ERFJ\nVpSJZDCwNWG7NNzX2OVmttIYtB9VAAAGxUlEQVTMnjSzoeG+0cB+M3vKzJab2d1hD6cvsN/d605y\nTRERaSNRJpKm3uDzRtsLgAJ3nwC8QNDDgOCx5POBm4BzgRHAnCSvGfzhZteZWZGZFZWVlZ169CIi\nkpQoE0kpMDRhewiwPbGAu+919+pw80FgcsK5y8NhsTrgD8A5wB6gj5llHu+aCdd+wN0L3b0wLy+v\nVSokIiL/KMoXEpcCo8xsOLANuAr4dGIBMxvk7jvCzdnA6oRzc8wsz93LCO6LFLm7m9lLwBUE91w+\nDzxzskCWLVu2x8w2N7Me/QgSWHvQXurSXuoBqkuqai91aWk98pMpZO5Njgy1CjO7BPgxkAE87O7f\nM7PbCZLCfDP7T4IEUgfsA6539zXhuRcB9xAMZy0DrnP3GjMbQZBEcoHlwNUJvZoo6lDk7oVRXb8t\ntZe6tJd6gOqSqtpLXdqqHpFOkeLui4BFjfbdkvB5LjD3OOcuASY0sX8DwRNhIiKSAvRmu4iItIgS\nyck9EHcArai91KW91ANUl1TVXurSJvWI9B6JiIi0f+qRiIhIiyiRhJKYYDI7nCSyJJw0sqDtozy5\nlkyUmWrM7GEz221mbx/nuJnZT8K6rjSzc9o6xmQkUY8PmllFwndyS1PlUoGZDTWzl8LJVIvN7KtN\nlEn57yXJeqTF92JmXczsdTN7M6zLbU2Uibb9cvcO/0PwePI7BG/QZwFvAuMalfln4Bfh56sIJpuM\nPfZm1GMO8LO4Y02yPhcQvIj69nGOXwI8R/CI+FTgtbhjbmY9PggsjDvOJOsyCDgn/NwTWNfE37GU\n/16SrEdafC/h/+ce4efOwGvA1EZlIm2/1CMJJDPB5KUcm8LlSeDDlnoLuTd7osxU5O6vELxfdDyX\nAo954FWCWQ8GtU10yUuiHmnD3Xe4+xvh5wMELxE3nu8u5b+XJOuRFsL/zwfDzc7hT+Ob35G2X0ok\ngWQmmDxaxoNpWyoIJpFMJS2ZKDMdJVvfdPC+cGjiOTMbH3cwyQiHRyYR/AacKK2+lxPUA9LkezGz\nDDNbQbDcxhJ3P+53EkX7pUQSSGYyyKQnjIxRSybKTEfp8J0k4w0g393PBn5KMLdcSjOzHsD/Av/q\n7pWNDzdxSkp+LyepR9p8L+5e7+4TCeYfnGJmZzYqEul3okQSOOkEk4llwkkje5N6wxUtmSgzHSXz\nvaU8d688MjThwWwQnc2sX8xhHZeZdSZofH/j7k81USQtvpeT1SPdvhcAd98P/BmY0ehQpO2XEkng\n6ASTFizdexUwv1GZ+QSTREIwaeSLHt65SiEnrUejserEiTLT0Xzgc+FTQlOBCj82CWjaMLOBR8ar\nzWwKwb/LvfFG1bQwzv8GVrv7j45TLOW/l2TqkS7fi5nlmVmf8HNX4CPAmkbFIm2/Ip1rK124e52Z\n3QD8kWMTTBZbwgSTBH/pfm1mJQSZ/Kr4Im5akvW40cwSJ8qcE1vAJ2FmjxM8OdPPzEqBWwluJOLu\nvyCYx+0SoASoAq6JJ9ITS6IeVwDXm1kdcAi4KgV/STliGvBZ4K1wTB7gm8AwSKvvJZl6pMv3Mgh4\n1ILF/zoBv3P3hW3ZfunNdhERaRENbYmISIsokYiISIsokYiISIsokYiISIsokYiISIsokYi0AjM7\nePJSJzz/STMbEX7uYWa/NLN3wtlcXzGz88wsK/ysx/YlpSiRiMQsnMMpw903hLseInjWf5S7jyd4\n16dfOBHnn4ArYwlU5DiUSERaUfg2991m9raZvWVmV4b7O5nZf4U9jIVmtsjMrghP+wzwTFjudOA8\n4Nvu3gAQzub8bFj2D2F5kZShLrJI6/onYCJwNtAPWGpmrxC8SV0AnAX0J5ia5uHwnGnA4+Hn8cAK\nd68/zvXfBs6NJHKRZlKPRKR1fQB4PJyNdRfwMkHD/wHg9+7e4O47gZcSzhkElCVz8TDB1JhZz1aO\nW6TZlEhEWtfxFgs60SJCh4Au4edi4GwzO9G/zWzgcDNiE4mEEolI63oFuDJcaCiPYJnd14G/ECwo\n1snMBhBM4njEamAkgLu/AxQBtyXMPDvKzC4NP/cFyty9tq0qJHIySiQiretpYCXwJvAi8O/hUNb/\nEqwJ8TbwS4LV+CrCc57lvYnlWmAgUGJmbxGsG3NkPY8PEcyuK5IyNPuvSBsxsx7ufjDsVbwOTHP3\nneEaEi+F28e7yX7kGk8Bc919bRuELJIUPbUl0nYWhgsQZQF3hD0V3P2Qmd1KsK72luOdHC5W9gcl\nEUk16pGIiEiL6B6JiIi0iBKJiIi0iBKJiIi0iBKJiIi0iBKJiIi0iBKJiIi0yP8HnGerC2h1l9wA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xeeff9b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "\n",
    "n_classes = 5\n",
    "\n",
    "array = lrcv_L2.scores_\n",
    "\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "   # print(np.mean(tuple(array.values())[0][j],axis = 0))\n",
    "    print tuple(array.values())[0][j]\n",
    "    scores[j][:] = tuple(array.values())[0][j]\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
